diff options
author | Bjørn Christian Seime <bjorn.christian@seime.no> | 2017-08-25 13:31:06 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-25 13:31:06 +0200 |
commit | d5d2098a6fa163e9a88f2ab09471ce6380b189f9 (patch) | |
tree | dd677f1aa2f89cfc5aad5dd223fd273c3a9c6274 | |
parent | c34d8ea5a1528c7f8098c1b61d0c7b8a4354fe1d (diff) | |
parent | 56aa0fadd0464e66ad80247e1d92bce5500b584d (diff) |
Merge branch 'master' into geirst/sort-document-types-in-topological-order
413 files changed, 22031 insertions, 422 deletions
diff --git a/.travis.yml b/.travis.yml index 6e03d4f73b2..29aa44a4e9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ branches: before_cache: - sudo rm -rf $HOME/.m2/repository/com/yahoo - du --summarize --human-readable $HOME/.m2/repository + - du --summarize --human-readable $HOME/.ccache - ccache --show-stats install: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 33fe2ea4f1a..ddc316e5b9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ refer to the [APIs](http://docs.vespa.ai/documentation/api.html). ## Where to start contributing Most features plug into the [Vespa Container](docs.vespa.ai/documentation/jdisc/index.html) - -this is the most likely pleace to write enhancements. +this is the most likely place to write enhancements. Discuss with the community if others have similar feature requests - make the feature generic. ### Getting started diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index fa2ee8e89a9..e59f012856a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -42,6 +42,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -79,7 +80,7 @@ public final class ConfiguredApplication implements Application { new ComponentRegistry<>(), new ComponentRegistry<>()); private final OsgiFramework restrictedOsgiFramework; - private volatile int applicationSerialNo = 0; + private final AtomicInteger applicationSerialNo = new AtomicInteger(0); private HandlersConfigurerDi configurer; private ScheduledThreadPoolExecutor shutdownDeadlineExecutor; private Thread reconfigurerThread; @@ -172,10 +173,12 @@ public final class ConfiguredApplication implements Application { startAndStopServers(); log.info("Switching to the latest deployed set of configurations and components. " + - "Application switch number: " + (applicationSerialNo++)); + "Application switch number: " + applicationSerialNo.getAndIncrement()); } private ContainerBuilder createBuilderWithGuiceBindings() { + log.info("Initializing new set of configurations and components. " + + "Application switch number: " + applicationSerialNo.get()); ContainerBuilder builder = activator.newContainerBuilder(); setupGuiceBindings(builder.guiceModules()); return builder; diff --git a/controller-api/OWNERS b/controller-api/OWNERS new file mode 100644 index 00000000000..e6a0537ba53 --- /dev/null +++ b/controller-api/OWNERS @@ -0,0 +1,2 @@ +bratseth +mpolden diff --git a/controller-api/pom.xml b/controller-api/pom.xml new file mode 100644 index 00000000000..51666da0c03 --- /dev/null +++ b/controller-api/pom.xml @@ -0,0 +1,181 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>controller-api</artifactId> + <packaging>container-plugin</packaging> + <version>6-SNAPSHOT</version> + + <dependencies> + + <!-- provided --> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <scope>provided</scope> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <scope>provided</scope> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <scope>provided</scope> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>serviceview</artifactId> + <scope>provided</scope> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <scope>provided</scope> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jdk8</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.glassfish.jersey.media</groupId> + <artifactId>jersey-media-multipart</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-server</artifactId> + <version>${jersey2.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <classifier>no_aop</classifier> + <scope>provided</scope> + </dependency> + + <!-- compile --> + + <dependency> + <groupId>com.intellij</groupId> + <artifactId>annotations</artifactId> + <version>9.0.4</version> + </dependency> + + <!-- test --> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>target/${project.artifactId}-deploy.jar</file> + <type>jar</type> + <classifier>deploy</classifier> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java new file mode 100644 index 00000000000..4233f6308d5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.yahoo.vespa.hosted.controller.api.application.v4.model.AthensDomainsResponse; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantInfo; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantPipelinesInfo; + + +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import java.util.List; + +/** + * @author gv + */ +@Path("/v4/") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public interface ApplicationApi { + + @GET + @Path(TenantResource.API_PATH) + List<TenantInfo> listTenants(); + + @Path(TenantResource.API_PATH + "/{tenantId}") + TenantResource tenant(@PathParam("tenantId")TenantId tenantId); + + @GET + @Path("athensDomain") + AthensDomainsResponse listAthensDomains(@DefaultValue("") @QueryParam("prefix") String prefix); + + @GET + @Path("tenant-pipeline") + TenantPipelinesInfo listTenantPipelines(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java new file mode 100644 index 00000000000..e5833682c90 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.yahoo.vespa.hosted.controller.api.application.v4.model.JobStatusList; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ApplicationReference; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.InstancesReply; + +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.List; + +/** + * @author gv + */ +@Path("") //Ensures that the produces annotation is inherited +@Produces(MediaType.APPLICATION_JSON) +public interface ApplicationResource { + + String API_PATH = "application"; + + @GET + List<ApplicationReference> listApplications(); + + @Path("{applicationId}") + @POST + ApplicationReference createApplication(@PathParam("applicationId") ApplicationId applicationId); + + @Path("{applicationId}") + @DELETE + void deleteApplication(@PathParam("applicationId") ApplicationId applicationId); + + @Path("{applicationId}/environment") + EnvironmentResource environment(); + + @Path("{applicationId}") + @GET + InstancesReply listInstances(@PathParam("applicationId") ApplicationId applicationId); + + @Path("{applicationId}/deployment") + @GET + JobStatusList deployment(@PathParam("applicationId") ApplicationId applicationId); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java new file mode 100644 index 00000000000..4f1583dd905 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java @@ -0,0 +1,98 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.fasterxml.jackson.databind.JsonNode; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployResult; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.InstanceInformation; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; +import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import java.io.InputStream; + +/** + * @author Tony Vaagenes + * @author gv + */ +@Path("") //Ensures that the produces annotation is inherited +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public interface EnvironmentResource { + + String API_PATH = "environment"; + + String APPLICATION_ZIP = "applicationZip"; + String DEPLOY_OPTIONS = "deployOptions"; + + @POST + @Path("{environmentId}/region/{regionId}/instance/{instanceId}/deploy") + @Consumes({MediaType.MULTIPART_FORM_DATA}) + DeployResult deploy(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId, + @FormDataParam(APPLICATION_ZIP) InputStream applicationZipFile, + @FormDataParam(APPLICATION_ZIP) FormDataContentDisposition fileMetaData, + @FormDataParam(DEPLOY_OPTIONS) FormDataBodyPart deployOptions); + + @DELETE + @Path("{environmentId}/region/{regionId}/instance/{instanceId}") + String deactivate(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId); + + @POST + @Path("{environmentId}/region/{regionId}/instance/{instanceId}/restart") + String restart(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId, + @QueryParam("hostname") Hostname hostname); + + @GET + @Path("{environmentId}/region/{regionId}/instance/{instanceId}") + InstanceInformation instanceInfo(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId); + + @GET + @Path("{environmentId}/region/{regionId}/instance/{instanceId}/converge") + JsonNode waitForConfigConverge(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId, + @QueryParam("timeout") long timeoutInSeconds); + + @POST + @Path("{environmentId}/region/{regionId}/instance/{instanceId}/log") + JsonNode grabLog(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId); + + @Path("{environmentId}/region/{regionId}/instance/{instanceId}/service") + ServiceViewResource service(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java new file mode 100644 index 00000000000..c058a72341a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.yahoo.vespa.serviceview.bindings.ApplicationView; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.HashMap; + +/** + * @author Stian Kristoffersen + */ +@Path("") +@Produces(MediaType.APPLICATION_JSON) +public interface ServiceViewResource { + + @GET + @Path("") + @Produces(MediaType.APPLICATION_JSON) + ApplicationView getUserInfo(); + + @GET + @Path("{serviceIdentifier}/{apiParams: .*}") + @Produces(MediaType.APPLICATION_JSON) + @SuppressWarnings("rawtypes") + HashMap singleService(@PathParam("serviceIdentifier") String identifier, + @PathParam("apiParams") String apiParams); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java new file mode 100644 index 00000000000..8db6f982ef6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java @@ -0,0 +1,47 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantCreateOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantInfo; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantMigrateOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantUpdateOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantWithApplications; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +/** + * @author Tony Vaagenes + */ +@Path("") //Ensures that the produces annotation is inherited +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public interface TenantResource { + + String API_PATH = "tenant"; + + @GET + TenantWithApplications metaData(); + + @DELETE + TenantInfo deleteTenant(); + + @POST + TenantInfo createTenant(TenantCreateOptions tenantOptions); + + @PUT + TenantInfo updateTenant(TenantUpdateOptions tenantOptions); + + @Path(ApplicationResource.API_PATH) + ApplicationResource application(); + + @PUT + @Path("migrateTenantToAthens") + TenantInfo migrateTenantToAthens(TenantMigrateOptions tenantOptions); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java new file mode 100644 index 00000000000..a290323a245 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.UserInfo; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import javax.ws.rs.GET; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + +/** + * @author gv + */ +@Path("/v4/user") +@Produces(MediaType.APPLICATION_JSON) +public interface UserResource { + @GET + @JsonInclude(value = JsonInclude.Include.NON_NULL) + UserInfo whoAmI(@QueryParam("userOverride") UserId userOverride); + + @PUT + void createUserTenant(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java new file mode 100644 index 00000000000..c542987e78f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; + +import java.net.URI; + +/** + * @author Stian Kristoffersen + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ApplicationReference { + public ApplicationId application; + public URI url; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java new file mode 100644 index 00000000000..400b973a4e1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.List; + +/** + * @author gv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AthensDomainsResponse extends JsonResponse<List<AthensDomain>> { + public AthensDomainsResponse(List<AthensDomain> athensDomainList) { + super(athensDomainList); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java new file mode 100644 index 00000000000..d8551898f7c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.component.Version; + +import java.util.Optional; + +/** + * @author gjoranv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DeployOptions { + + public final Optional<ScrewdriverBuildJob> screwdriverBuildJob; + public final Optional<String> vespaVersion; + public final boolean ignoreValidationErrors; + public final boolean deployCurrentVersion; + + @JsonCreator + public DeployOptions(@JsonProperty("screwdriverBuildJob") Optional<ScrewdriverBuildJob> screwdriverBuildJob, + @JsonProperty("vespaVersion") Optional<Version> vespaVersion, + @JsonProperty("ignoreValidationErrors") boolean ignoreValidationErrors, + @JsonProperty("deployCurrentVersion") boolean deployCurrentVersion) { + this.screwdriverBuildJob = screwdriverBuildJob; + this.vespaVersion = vespaVersion.map(Version::toString); + this.ignoreValidationErrors = ignoreValidationErrors; + this.deployCurrentVersion = deployCurrentVersion; + } + + @Override + public String toString() { + return "DeployData{" + + "screwdriverBuildJob=" + screwdriverBuildJob.map(ScrewdriverBuildJob::toString).orElse("None") + + ", vespaVersion=" + vespaVersion.orElse("None") + + ", ignoreValidationErrors=" + ignoreValidationErrors + + ", deployCurrentVersion=" + deployCurrentVersion + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java new file mode 100644 index 00000000000..3a98926805f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; +import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; + +import java.util.List; + +/** + * @author gjoranv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DeployResult { + + public final RevisionId revisionId; + public final Long applicationZipSize; + public final List<LogEntry> prepareMessages; + public final ConfigChangeActions configChangeActions; + + @JsonCreator + public DeployResult(@JsonProperty("revisionId") RevisionId revisionId, + @JsonProperty("applicationZipSize") Long applicationZipSize, + @JsonProperty("prepareMessages") List<LogEntry> prepareMessages, + @JsonProperty("configChangeActions") ConfigChangeActions configChangeActions) { + this.revisionId = revisionId; + this.applicationZipSize = applicationZipSize; + this.prepareMessages = prepareMessages; + this.configChangeActions = configChangeActions; + } + + @Override + public String toString() { + return "DeployResult{" + + "revisionId=" + revisionId.id() + + ", applicationZipSize=" + applicationZipSize + + ", prepareMessages=" + prepareMessages + + ", configChangeActions=" + configChangeActions + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java new file mode 100644 index 00000000000..d014a82bf62 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java @@ -0,0 +1,58 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +/** + * Represent the operational status of a service endpoint (where the endpoint itself + * is identified by the container cluster id). + * + * The status of an endpoint may be assigned from the controller. + * + * @author smorgrav + */ +public class EndpointStatus { + private final String agent; + private final String reason; + private final Status status; + private final long epoch; + + public enum Status { + in, + out, + unknown; + } + + public EndpointStatus(Status status, String reason, String agent, long epoch) { + this.status = status; + this.reason = reason; + this.agent = agent; + this.epoch = epoch; + } + + /** + * @return The agent responsible setting this status + */ + public String getAgent() { + return agent; + } + + /** + * @return The reason for this status (e.g. 'incident INCXXX') + */ + public String getReason() { + return reason; + } + + /** + * @return The current status + */ + public Status getStatus() { + return status; + } + + /** + * @return The epoch for when this status became active + */ + public long getEpoch() { + return epoch; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java new file mode 100644 index 00000000000..317da739103 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; + +import java.util.Objects; + +/** + * @author gv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class GitRevision { + + public final GitRepository repository; + public final GitBranch branch; + public final GitCommit commit; + + @JsonCreator + public GitRevision(@JsonProperty("repository") GitRepository repository, + @JsonProperty("branch") GitBranch branch, + @JsonProperty("commit") GitCommit commit) { + this.repository = repository; + this.branch = branch; + this.commit = commit; + } + + @Override + public String toString() { + return "GitRevision{" + + "repository=" + repository + + ", branch=" + branch + + ", commit=" + commit + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GitRevision that = (GitRevision) o; + return Objects.equals(repository, that.repository) && + Objects.equals(branch, that.branch) && + Objects.equals(commit, that.commit); + } + + @Override + public int hashCode() { + return Objects.hash(repository, branch, commit); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java new file mode 100644 index 00000000000..e862bd744dc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.yahoo.vespa.hosted.controller.api.cost.CostJsonModel; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; +import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; + +import java.net.URI; +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class InstanceInformation { + public List<URI> serviceUrls; + public URI nodes; + public URI elkUrl; + public URI yamasUrl; + public RevisionId revision; + public Long deployTimeEpochMs; + public Long expiryTimeEpochMs; + + public ScrewdriverId screwdriverId; + public GitRepository gitRepository; + public GitBranch gitBranch; + public GitCommit gitCommit; + + public CostJsonModel.Application cost; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java new file mode 100644 index 00000000000..6ac27d0bad9 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.yahoo.vespa.hosted.controller.api.bcp.BcpStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; +import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; + +import java.net.URI; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class InstanceReference { + public EnvironmentId environment; + public RegionId region; + public InstanceId instance; + public BcpStatus bcpStatus; + + public URI url; + + public static InstanceReference createInstanceReference(InstanceId instanceId, RegionId regionId, EnvironmentId environmentId, URI uri) { + InstanceReference instanceReference = new InstanceReference(); + instanceReference.instance = instanceId; + instanceReference.region = regionId; + instanceReference.environment = environmentId; + instanceReference.url = uri; + return instanceReference; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java new file mode 100644 index 00000000000..ff0c155460e --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.net.URI; +import java.util.List; +import java.util.Set; + +/** + * @author Tony Vaagenes + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class InstancesReply { + public Set<URI> globalRotations; + public List<InstanceReference> instances; + public String compileVersion; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java new file mode 100644 index 00000000000..cce8a8c88fc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author bratseth + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class JobStatus { + + public String jobType; + public long lastCompleted; + public boolean success; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java new file mode 100644 index 00000000000..30af3291fbd --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.List; + +/** + * @author bratseth + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class JobStatusList { + public List<JobStatus> jobs; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java new file mode 100644 index 00000000000..3690644c49b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +/** + * @author gv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(Include.NON_NULL) +public abstract class JsonResponse<DATA> { + public DATA data; + + public JsonResponse(DATA data) { + this.data = data; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java new file mode 100644 index 00000000000..d5fc0addd70 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author gjoranv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class LogEntry { + + public final long time; + public final String level; + public final String message; + + @JsonCreator + public LogEntry(@JsonProperty("time") long time, + @JsonProperty("level") String level, + @JsonProperty("message") String message) { + this.time = time; + this.level = level; + this.message = message; + } + + @Override + public String toString() { + return "LogEntry{" + + "time=" + time + + ", level='" + level + '\'' + + ", message='" + message + '\'' + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java new file mode 100644 index 00000000000..032b97c5424 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java @@ -0,0 +1,47 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; + +import java.util.Objects; + +/** + * @author gjoranv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ScrewdriverBuildJob { + public final ScrewdriverId screwdriverId; + public final GitRevision gitRevision; + + @JsonCreator + public ScrewdriverBuildJob(@JsonProperty("screwdriverId") ScrewdriverId screwdriverId, + @JsonProperty("gitRevision") GitRevision gitRevision) { + this.screwdriverId = screwdriverId; + this.gitRevision = gitRevision; + } + + @Override + public String toString() { + return "ScrewdriverBuildJob{" + + "screwdriverId=" + screwdriverId + + ", gitRevision=" + gitRevision + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ScrewdriverBuildJob that = (ScrewdriverBuildJob) o; + return Objects.equals(screwdriverId, that.screwdriverId) && + Objects.equals(gitRevision, that.gitRevision); + } + + @Override + public int hashCode() { + return Objects.hash(screwdriverId, gitRevision); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java new file mode 100644 index 00000000000..4032a960b3c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = JsonInclude.Include.NON_NULL) +public class TenantCreateOptions { + public AthensDomain athensDomain; + public Property property; + public PropertyId propertyId; + public UserGroup userGroup; + + public TenantCreateOptions() {} + + public TenantCreateOptions(UserGroup userGroup, Property property, PropertyId propertyId) { + this.userGroup = userGroup; + this.property = property; + this.propertyId = propertyId; + } + + public TenantCreateOptions(AthensDomain athensDomain, Property property, PropertyId propertyId) { + this.athensDomain = athensDomain; + this.property = property; + this.propertyId = propertyId; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("options: "); + sb.append("athens-domain='").append(this.athensDomain).append("', "); + sb.append("property='").append(this.property).append("'"); + if (this.propertyId != null) { + sb.append(", propertyId='").append(this.propertyId).append("'"); + } + + return sb.toString(); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java new file mode 100644 index 00000000000..ef1afbc9edf --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; + +import java.net.URI; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +public class TenantInfo { + public TenantId tenant; + // TODO: make optional + public TenantMetaData metaData; + public URI url; + + // Required for Jackson deserialization + public TenantInfo() {} + + public TenantInfo(TenantId tenantId, TenantMetaData metaData, URI url) { + this.tenant = tenantId; + this.metaData = metaData; + this.url = url; + } + + public TenantInfo(TenantId tenant, URI url) { + this.tenant = tenant; + this.url = url; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java new file mode 100644 index 00000000000..5ded1d8030e --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; + +import java.util.Optional; + +/** + * @author gv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = Include.NON_EMPTY) +public class TenantMetaData { + public TenantType type; + public Optional<AthensDomain> athensDomain; + public Optional<Property> property; + public Optional<UserGroup> userGroup; + + // Required for Jackson deserialization + public TenantMetaData() {} + + public TenantMetaData(TenantType type, + Optional<AthensDomain> athensDomain, + Optional<Property> property, + Optional<UserGroup> userGroup) { + this.type = type; + this.athensDomain = athensDomain; + this.property = property; + this.userGroup = userGroup; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java new file mode 100644 index 00000000000..9c748eafd38 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = JsonInclude.Include.NON_NULL) +public class TenantMigrateOptions { + + public AthensDomain athensDomain; + + public TenantMigrateOptions() {} + + public TenantMigrateOptions(AthensDomain athensDomain) { + this.athensDomain = athensDomain; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java new file mode 100644 index 00000000000..a7f1fb408fe --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class TenantPipelinesInfo { + public List<TenantPipelineInfo> tenantPipelines = new ArrayList<>(); + + public static class TenantPipelineInfo { + public String screwdriverId; + public String tenant; + public String application; + public String instance; + } + + public List<TenantPipelineInfo> brokenTenantPipelines = new ArrayList<>(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java new file mode 100644 index 00000000000..2c543af7bf8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +/** + * @author bjorncs + */ +public enum TenantType { + OPSDB, + USER, + ATHENS +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java new file mode 100644 index 00000000000..9b2f24e2f62 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java @@ -0,0 +1,58 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; + +import java.util.Objects; +import java.util.Optional; + +/** + * @author gv + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = JsonInclude.Include.NON_ABSENT) +public class TenantUpdateOptions { + public final Property property; + public final Optional<UserGroup> userGroup; + public final Optional<AthensDomain> athensDomain; + + @JsonCreator + public TenantUpdateOptions(@JsonProperty("property") Property property, + @JsonProperty("userGroup") Optional<UserGroup> userGroup, + @JsonProperty("athensDomain") Optional<AthensDomain> athensDomain) { + this.userGroup = userGroup; + this.property = property; + this.athensDomain = athensDomain; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TenantUpdateOptions that = (TenantUpdateOptions) o; + return Objects.equals(property, that.property) && + Objects.equals(userGroup, that.userGroup) && + Objects.equals(athensDomain, that.athensDomain); + } + + @Override + public int hashCode() { + return Objects.hash(property, userGroup, athensDomain); + } + + @Override + public String toString() { + return "TenantUpdateOptions{" + + "property=" + property + + ", userGroup=" + userGroup + + ", athensDomain=" + athensDomain + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java new file mode 100644 index 00000000000..de731d5c971 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java @@ -0,0 +1,39 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; + +import java.util.List; + +/** + * @author Tony Vaagenes + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(value = JsonInclude.Include.NON_NULL) +public class TenantWithApplications { + // TODO: use TenantMetaData instead of individual fields (requires dashboard updates) + public TenantType type; + public AthensDomain athensDomain; + public Property property; + public UserGroup userGroup; + public List<ApplicationReference> applications; + + public TenantWithApplications() {} + + public TenantWithApplications( + TenantType type, + AthensDomain athensDomain, + Property property, + UserGroup userGroup, + List<ApplicationReference> applications) { + this.type = type; + this.athensDomain = athensDomain; + this.property = property; + this.userGroup = userGroup; + this.applications = applications; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java new file mode 100644 index 00000000000..2b2a089c543 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.util.List; + +/** + * @author gv + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class UserInfo { + public UserId user; + public boolean tenantExists; + public List<TenantInfo> tenants; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java new file mode 100644 index 00000000000..397461a829d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfigChangeActions { + @JsonProperty("restart") public final List<RestartAction> restartActions; + @JsonProperty("refeed") public final List<RefeedAction> refeedActions; + + @JsonCreator + public ConfigChangeActions(@JsonProperty("restart") List<RestartAction> restartActions, + @JsonProperty("refeed") List<RefeedAction> refeedActions) { + this.restartActions = restartActions; + this.refeedActions = refeedActions; + } + + @Override + public String toString() { + return "ConfigChangeActions{" + + "restartActions=" + restartActions + + ", refeedActions=" + refeedActions + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java new file mode 100644 index 00000000000..0546a3b5c44 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RefeedAction { + public final String name; + public final boolean allowed; + public final String documentType; + public final String clusterName; + public final List<ServiceInfo> services; + public final List<String> messages; + + @JsonCreator + public RefeedAction(@JsonProperty("name") String name, + @JsonProperty("allowed") boolean allowed, + @JsonProperty("documentType") String documentType, + @JsonProperty("clusterName") String clusterName, + @JsonProperty("services") List<ServiceInfo> services, + @JsonProperty("messages") List<String> messages) { + this.name = name; + this.allowed = allowed; + this.documentType = documentType; + this.clusterName = clusterName; + this.services = services; + this.messages = messages; + } + + @Override + public String toString() { + return "RefeedAction{" + + "name='" + name + '\'' + + ", allowed=" + allowed + + ", documentType='" + documentType + '\'' + + ", clusterName='" + clusterName + '\'' + + ", services=" + services + + ", messages=" + messages + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java new file mode 100644 index 00000000000..a760a26d47d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RestartAction { + public final String clusterName; + public final String clusterType; + public final String serviceType; + public final List<ServiceInfo> services; + public final List<String> messages; + + @JsonCreator + public RestartAction(@JsonProperty("clusterName") String clusterName, + @JsonProperty("clusterType") String clusterType, + @JsonProperty("serviceType") String serviceType, + @JsonProperty("services") List<ServiceInfo> services, + @JsonProperty("messages") List<String> messages) { + this.clusterName = clusterName; + this.clusterType = clusterType; + this.serviceType = serviceType; + this.services = services; + this.messages = messages; + } + + @Override + public String toString() { + return "RestartAction{" + + "clusterName='" + clusterName + '\'' + + ", clusterType='" + clusterType + '\'' + + ", serviceType='" + serviceType + '\'' + + ", services=" + services + + ", messages=" + messages + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java new file mode 100644 index 00000000000..8d03d2da440 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author bjorncs + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ServiceInfo { + public final String serviceName; + public final String serviceType; + public final String configId; + public final String hostName; + + @JsonCreator + public ServiceInfo(@JsonProperty("serviceName") String serviceName, + @JsonProperty("serviceType") String serviceType, + @JsonProperty("configId") String configId, + @JsonProperty("hostName")String hostName) { + this.serviceName = serviceName; + this.serviceType = serviceType; + this.configId = configId; + this.hostName = hostName; + } + + @Override + public String toString() { + return "ServiceInfo{" + + "serviceName='" + serviceName + '\'' + + ", serviceType='" + serviceType + '\'' + + ", configId='" + configId + '\'' + + ", hostName='" + hostName + '\'' + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java new file mode 100644 index 00000000000..1201f148329 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java new file mode 100644 index 00000000000..1eac6d8c296 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.application.v4.model; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java new file mode 100644 index 00000000000..e7b71b693a3 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.application.v4; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java new file mode 100644 index 00000000000..679d5fc5727 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.bcp; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class BcpStatus { + public String rotationStatus; + public String reason; + + // For jackson + public BcpStatus() {} + + public BcpStatus(String rotationStatus, String reason) { + this.rotationStatus = rotationStatus; + this.reason = reason; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java new file mode 100644 index 00000000000..c77f9fceef9 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.bcp; + +import com.fasterxml.jackson.databind.JsonNode; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +/** + * @author andreer + */ +@Path("") //Ensures that the produces annotation is inherited +@Produces(MediaType.APPLICATION_JSON) +public interface BrooklynStatusResource { + + @GET + @Path("{rotation}") + @Produces(MediaType.APPLICATION_JSON) + JsonNode rotationStatus(@PathParam("rotation") String page); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java new file mode 100644 index 00000000000..2bb442c3db8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.bcp; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java new file mode 100644 index 00000000000..ad28d3ca5b5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.configserver; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * Environment representation using the same definition as configserver. And allowing + * serialization/deserialization to/from JSON. + * + * @author Ulf Lilleengen + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Environment { + private final com.yahoo.config.provision.Environment environment; + + public Environment(com.yahoo.config.provision.Environment environment) { + this.environment = environment; + } + + @JsonValue + public String value() { + return environment.value(); + } + + @Override + public String toString() { + return value(); + } + + public com.yahoo.config.provision.Environment getEnvironment() { + return environment; + } + + public Environment(String environment) { + this.environment = com.yahoo.config.provision.Environment.from(environment); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java new file mode 100644 index 00000000000..b7f1560eb67 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java @@ -0,0 +1,39 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.configserver; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonValue; +import com.yahoo.config.provision.RegionName; + +/** + * Region representation using the same definition as configserver. And allowing + * serialization/deserialization to/from JSON. + * + * @author Ulf Lilleengen + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Region { + private final RegionName region; + + public Region(RegionName region) { + this.region = region; + } + + @JsonValue + public String value() { + return region.value(); + } + + @Override + public String toString() { return value(); } + + public RegionName getRegion() { + return region; + } + + @JsonCreator + public Region(String region) { + this.region = com.yahoo.config.provision.RegionName.from(region); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java new file mode 100644 index 00000000000..f035e200661 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.configserver; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java new file mode 100644 index 00000000000..bfc451946f6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.cost; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +/** + * JSON datamodel for the cost api. + * + * @author smorgrav + */ +public class CostJsonModel { + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Application { + + @JsonProperty + public String zone; + @JsonProperty + public String tenant; + @JsonProperty + public String app; + @JsonProperty + public int tco; + @JsonProperty + public float utilization; + @JsonProperty + public float waste; + @JsonProperty + public Map<String, Cluster> cluster; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Cluster { + + @JsonProperty + public int count; + @JsonProperty + public String resource; + @JsonProperty + public float utilization; + @JsonProperty + public int tco; + @JsonProperty + public String flavor; + @JsonProperty + public int waste; + @JsonProperty + public String type; + @JsonProperty + public HardwareResources util; + @JsonProperty + public HardwareResources usage; + @JsonProperty + public List<String> hostnames; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class HardwareResources { + + @JsonProperty + public float mem; + @JsonProperty + public float disk; + @JsonProperty + public float cpu; + @JsonProperty("diskbusy") + public float diskBusy; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java new file mode 100644 index 00000000000..3cc6d682f4a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java @@ -0,0 +1,41 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.cost; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.List; + +/** + * Cost and Utilization APi for hosted Vespa. + * + * Used to give insight to PEG and application owners about + * TOC and if the application is reasonable scaled. + * + * @author smorgrav + */ +@Path("v1") +@Produces(MediaType.APPLICATION_JSON) +public interface CostResource { + + @GET + @Path("/analysis/cpu") + List<CostJsonModel.Application> getCPUAnalysis(); + + @GET + @Produces("text/csv") + @Path("/csv") + String getCSV(); + + @GET + @Path("/apps") + List<CostJsonModel.Application> getApplicationsCost(); + + @GET + @Path("/apps/{environment}/{region}/{application}") + CostJsonModel.Application getApplicationCost(@PathParam("application") String appName, + @PathParam("region") String regionName, + @PathParam("environment") String envName); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java new file mode 100644 index 00000000000..8e95bd4f6f1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.cost; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java new file mode 100644 index 00000000000..0a5f2809780 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class ApplicationId extends NonDefaultIdentifier { + + public ApplicationId(String id) { + super(id); + } + + public static boolean isLegal(String id) { + return strictPattern.matcher(id).matches(); + } + + @Override + public void validate() { + super.validate(); + validateNoUpperCase(); + } + + public static void validate(String id) { + if (!isLegal(id)) { + throwInvalidId(id, "Must match pattern " + strictPattern, "application"); + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java new file mode 100644 index 00000000000..eb8b5c5256b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class AthensDomain extends Identifier { + + public AthensDomain(String id) { + super(id); + } + + public boolean isTopLevelDomain() { + return !id().contains("."); + } + + public AthensDomain getParent() { + return new AthensDomain(id().substring(0, lastDot())); + } + + public String getName() { + return id().substring(lastDot() + 1); + } + + private int lastDot() { + return id().lastIndexOf('.'); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java new file mode 100644 index 00000000000..80fe98a4489 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.yahoo.config.provision.Zone; + +/** + * Application + zone. + * + * @author smorgrav + * @author bratseth + */ +public class DeploymentId { + + private final com.yahoo.config.provision.ApplicationId application; + private final Zone zone; + + public DeploymentId(com.yahoo.config.provision.ApplicationId application, Zone zone) { + this.application = application; + this.zone = zone; + } + + public com.yahoo.config.provision.ApplicationId applicationId() { + return application; + } + public Zone zone() { return zone; } + + + public String dottedString() { + return unCapitalize(applicationId().tenant().value()) + "." + + unCapitalize(applicationId().application().value()) + "." + + unCapitalize(zone.environment().value()) + "." + + unCapitalize(zone.region().value()) + "." + + unCapitalize(application.instance().value()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DeploymentId other = (DeploymentId) o; + if ( ! this.application.equals(other.application)) return false; + // TODO: Simplify when Zone implements equals + if ( ! this.zone.environment().equals(other.zone.environment())) return false; + if ( ! this.zone.region().equals(other.zone.region())) return false; + return true; + } + + @Override + public int hashCode() { + // TODO: Simplify when Zone implements hashCode + return application.hashCode() + + 7 * zone.environment().hashCode() + + 31 * zone.region().hashCode(); + } + + @Override + public String toString() { + return toUserFriendlyString(); + } + + public String toUserFriendlyString() { + return application + " in " + zone; + } + + private static String unCapitalize(String str) { + return str.toLowerCase().substring(0,1) + str.substring(1); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java new file mode 100644 index 00000000000..a09e802c251 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class EnvironmentId extends NonDefaultIdentifier { + + public EnvironmentId(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java new file mode 100644 index 00000000000..31402825d3c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class GitBranch extends Identifier { + + public GitBranch(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java new file mode 100644 index 00000000000..289b3ec59a0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class GitCommit extends Identifier { + + public GitCommit(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java new file mode 100644 index 00000000000..50bbc0bd9f9 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class GitRepository extends Identifier { + + public GitRepository(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java new file mode 100644 index 00000000000..3f7437c5d0b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class Hostname extends Identifier { + + public Hostname(String hostname) { + super(hostname); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java new file mode 100644 index 00000000000..70ebc8712d5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java @@ -0,0 +1,103 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * @author smorgrav + */ +public abstract class Identifier { + + protected static final Pattern strictPattern = Pattern.compile("[a-z0-9][a-z0-9-]{0,26}[a-z0-9]"); + private static final Pattern serializedIdentifierPattern = Pattern.compile("[a-zA-Z0-9_-]+"); + private static final Pattern serializedPattern = Pattern.compile("[a-zA-Z0-9_.-]+"); + + private final String id; + + @JsonCreator + public Identifier(String id) { + Objects.requireNonNull(id, "Id string cannot be null"); + this.id = id; + validate(); + } + + public String toDns() { + return id.replace('_', '-'); + } + + @Override + public String toString() { + return id; + } + + @JsonValue + public String id() { return id; } + + public String capitalizedType() { + String simpleName = this.getClass().getSimpleName(); + String suffix = "Id"; + if (simpleName.endsWith(suffix)) { + simpleName = simpleName.substring(0, simpleName.length() - suffix.length()); + } + return simpleName; + } + + public void validate() { + if (id.equals("api")) { + throwInvalidId(id, "'api' not allowed."); + } + } + + protected void validateSerialized() { + if (!serializedPattern.matcher(id).matches()) { + throwInvalidId(id, "Must match pattern " + serializedPattern); + } + } + + protected void validateSerializedIdentifier() { + if (!serializedIdentifierPattern.matcher(id).matches()) { + throwInvalidId(id, "Must match pattern " + serializedIdentifierPattern); + } + } + + protected void validateNoDefault() { + if (id.equals("default")) { + throwInvalidId(id, "'default' not allowed."); + } + } + + protected void validateNoUpperCase() { + if (!id.equals(id.toLowerCase())) + throwInvalidId(id, "Uppercase not allowed."); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Identifier identity = (Identifier) o; + + return id.equals(identity.id); + + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + public static void throwInvalidId(String id, String explanation) { + throw new IllegalArgumentException(String.format("Invalid id: %s. %s", id, explanation)); + } + + public static void throwInvalidId(String id, String explanation, String idName) { + throw new IllegalArgumentException(String.format("Invalid %s id: %s. %s", idName, id, explanation)); + } + +} + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java new file mode 100644 index 00000000000..6e3087cdcf6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java @@ -0,0 +1,19 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class InstanceId extends SerializedIdentifier { + + public InstanceId(String id) { + super(id); + } + + @Override + public void validate() { + super.validate(); + validateNoUpperCase(); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java new file mode 100644 index 00000000000..96f0a9c43f0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * TODO: Class description + * + * @author smorgrav + */ +public abstract class NonDefaultIdentifier extends SerializedIdentifier { + + public NonDefaultIdentifier(String id) { + super(id); + } + + @Override + public void validate() { + super.validate(); + validateNoDefault(); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java new file mode 100644 index 00000000000..7dde9002310 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * A business property. + * + * @author smorgrav + */ +public class Property extends Identifier { + + public Property(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java new file mode 100644 index 00000000000..c84cfb9b512 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import java.util.regex.Pattern; + +/** + * A business property ID. + * + * @author frodelu + */ +public class PropertyId extends Identifier { + + private static final Pattern PATTERN = Pattern.compile("\\d+"); + + public PropertyId(String id) { + super(id); + } + + /** Returns this id as a long */ + public long value() { return Long.parseLong(id()); } + + @Override + public void validate() { + super.validate(); + if(!PATTERN.matcher(id()).matches()) { + throwInvalidId(id(), "Property id must match pattern: " + PATTERN); + } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java new file mode 100644 index 00000000000..bb6208ff8e3 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class RegionId extends NonDefaultIdentifier { + + public RegionId(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java new file mode 100644 index 00000000000..11094c69707 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * An unique identifier of an application package. + * + * @author smorgrav + */ +public class RevisionId extends SerializedIdentifier { + + public RevisionId(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java new file mode 100644 index 00000000000..aab18595d20 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class RotationId extends Identifier { + + public RotationId(String id) { + super(id); + } + + @Override + public void validate() { + super.validate(); + validateSerialized(); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java new file mode 100644 index 00000000000..b0fb72662c6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java @@ -0,0 +1,30 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import java.util.regex.Pattern; + +/** + * @author smorgrav + * @author bjorncs + */ +public class ScrewdriverId extends Identifier { + + // TODO: If only there was a separate type for this ... + // This demonstrates why this subclassing scheme is a bad idea + private static final Pattern PATTERN = Pattern.compile("\\d+"); + + public ScrewdriverId(String id) { + super(id); + } + + /** Returns this id as a long */ + public long value() { return Long.parseLong(id()); } + + @Override + public void validate() { + super.validate(); + if(!PATTERN.matcher(id()).matches()) { + throwInvalidId(id(), "Screwdriver id must match pattern: " + PATTERN); + } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java new file mode 100644 index 00000000000..3660262f9c1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * TODO: Class description + * + * @author smorgrav + */ + +public abstract class SerializedIdentifier extends Identifier { + + public SerializedIdentifier(String id) { + super(id); + } + + @Override + public void validate() { + super.validate(); + validateSerializedIdentifier(); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java new file mode 100644 index 00000000000..82cd6d80ec8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class TenantId extends NonDefaultIdentifier { + + public TenantId(String id) { + super(id); + } + + public boolean isUser() { + return id().startsWith("by-"); + } + + @Override + public void validate() { + super.validate(); + validateNoUpperCase(); + } + + public static void validate(String id) { + if (!strictPattern.matcher(id).matches()) { + throwInvalidId(id, "Must match pattern " + strictPattern, "tenant"); + } + } + + /** Return true if this is the user tenant of the given user */ + public boolean isTenantFor(UserId userId) { + return id().equals("by-" + userId.id().replace('_', '-')); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java new file mode 100644 index 00000000000..b6b0379bc90 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class UserGroup extends Identifier { + + public UserGroup(String id) { + super(id); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java new file mode 100644 index 00000000000..d2effc76827 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class UserId extends NonDefaultIdentifier { + + public UserId(String id) { + super(id); + } + + public TenantId toTenantId() { + return new TenantId("by-" + id().replace('_', '-')); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java new file mode 100644 index 00000000000..79210143d19 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +/** + * @author smorgrav + */ +public class ZoneId extends Identifier { + + public ZoneId(EnvironmentId envId, RegionId regionId) { + super(envId.id() + ":" + regionId.id()); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java new file mode 100644 index 00000000000..211a2ab7fc0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java new file mode 100644 index 00000000000..bbd15707cde --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +/** + * @author jvenstad + */ +public interface BuildService { + + /** + * Enqueue a job defined by "buildJob in an external build system, and return the outcome of the enqueue request. + * This method should return @false only when a retry is in order, and @true otherwise, e.g., on succes, or for invalid jobs. + */ + boolean trigger(BuildJob buildJob); + + class BuildJob { + + private final long projectId; + private final String jobName; + + public BuildJob(long projectId, String jobName) { + this.projectId = projectId; + this.jobName = jobName; + } + + public long projectId() { return projectId; } + public String jobName() { return jobName; } + + @Override + public String toString() { return jobName + "@" + projectId; } + + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java new file mode 100644 index 00000000000..329483a85c5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java @@ -0,0 +1,136 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.Collection; +import java.util.Objects; + +import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.unknown; +import static java.util.Comparator.reverseOrder; + +/** + * @author jvenstad + */ +public interface Contacts { + + /** + * Returns the most relevant user of lowest non-empty level above that of @assignee, or, if no such user exists, + * the @assignee with @Category information. + */ + static UserContact escalationTargetFrom(Collection<UserContact> userContacts, String assignee) { + return userContacts.stream() + .filter(contact -> ! contact.username().isEmpty()) // don't assign to empty names + .sorted(reverseOrder()).distinct() // Pick out the highest category per user. + // Keep the assignee, or the last user on the first non-empty level above her. + .sorted().reduce(new UserContact(assignee, assignee, unknown), (current, next) -> + next.is(assignee) || (current.is(assignee) ^ current.level() == next.level()) ? next : current); + } + + /** + * Return a list of all contact entries for property with id @propertyId, where username is set. + */ + Collection<UserContact> userContactsFor(long propertyId); + + /** Returns the URL listing contacts for the given property */ + URI contactsUri(long propertyId); + + /** + * Return a target of escalation above @assignee, from the set of @UserContact entries found for @propertyId. + */ + default UserContact escalationTargetFor(long propertyId, String assignee) { + return escalationTargetFrom(userContactsFor(propertyId), assignee); + } + + /** + * A list of contact roles, in the order in which we look for escalation targets. + * Categories must be listed in increasing order of relevancy per level, and by increasing level. + */ + enum Category { + + unknown(-1, Level.none, "Unknown"), + admin(54, Level.grunt, "Administrator"), // TODO: Find more grunts? + businessOwner(567, Level.owner, "Business Owner"), + serviceOwner(646, Level.owner, "Service Engineering Owner"), + engineeringOwner(566, Level.owner, "Engineering Owner"), + vpBusiness(11, Level.VP, "VP Business"), + vpService(647, Level.VP, "VP Service Engineering"), + vpEngineering(9, Level.VP, "VP Engineering"); + + public final long id; + public final Level level; + public final String name; + + Category(long id, Level level, String name) { + this.id = id; + this.level = level; + this.name = name; + } + + /** Find the category for the given id, or unknown if the id is unknown. */ + public static Category of(Long id) { + for (Category category : values()) + if (category.id == id) + return category; + return unknown; + } + + public enum Level { + none, + grunt, + owner, + VP; + } + + } + + /** Container class for user contact information; sorts by category and identifies by username. Immutable. */ + class UserContact implements Comparable<UserContact> { + + private final String username; + private final String name; + private final Category category; + + public UserContact(String username, String name, Category category) { + Objects.requireNonNull(username, "username cannot be null"); + Objects.requireNonNull(name, "name cannot be null"); + Objects.requireNonNull(category, "category cannot be null"); + this.username = username; + this.name = name; + this.category = category; + } + + public String username() { return username; } + public String name() { return name; } + public Category category() { return category; } + public Category.Level level() { return category.level; } + + public boolean is(String username) { return this.username.equals(username); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UserContact that = (UserContact) o; + return Objects.equals(username, that.username); + } + + @Override + public int hashCode() { + return Objects.hash(username); + } + + @Override + public int compareTo(@NotNull UserContact other) { + return category().compareTo(other.category()); + } + + @Override + public String toString() { + return String.format("%s, %s, %s", username, name, category.name); + } + + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java new file mode 100644 index 00000000000..6b7464b9ed0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java @@ -0,0 +1,182 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +/** + * @author jvenstad + */ +public interface Issues { + + /** + * Returns information about an issue. + * If this issue does not exist this returns an issue id containing the id and default values. + */ + IssueInfo fetch(String issueId); + + /** + * Returns the @Meta of all unresolved issues which have the same summary (and queue, if present) as @issue. + */ + List<IssueInfo> fetchSimilarTo(Issue issue); + + /** + * Files the given issue + * + * @return the id of the created issue + */ + String file(Issue issue); + + /** + * Update the description fields of the issue stored with id @issueId to be @description. + */ + void update(String issueId, String description); + + /** + * Set the assignee of the issue with id @issueId to the user with usename @assignee. + */ + void reassign(String issueId, String assignee); + + /** + * Add the user with username @watcher to the watcher list of the issue with id @issueId. + */ + void addWatcher(String issueId, String watcher); + + /** + * Post @comment as a comment to the issue with id @issueId. + */ + void comment(String issueId, String comment); + + + /** Contains information used to file an issue with the responsible party; only @queue is mandatory. */ + class Classification { + + private final String queue; + private final String component; + private final String label; + + public Classification(String queue, String component, String label) { + if (queue.isEmpty()) throw new IllegalArgumentException("Queue can not be empty!"); + + this.queue = queue; + this.component = component; + this.label = label; + } + + public Classification(String queue) { + this(queue, null, null); + } + + public Classification withComponent(String component) { return new Classification(queue, component, label); } + public Classification withLabel(String label) { return new Classification(queue, component, label); } + + public String queue() { return queue; } + public Optional<String> component() { return Optional.ofNullable(component); } + public Optional<String> label() { return Optional.ofNullable(label); } + + @Override + public String toString() { + return + "Queue : " + queue() + "\n" + + "Component : " + component() + "\n" + + "Label : " + label() + "\n"; + } + + } + + + /** Information about a stored issue */ + class IssueInfo { + + private final String id; + private final String key; + private final Instant updated; + private final Optional<String> assignee; + private final Status status; + + public IssueInfo(String id, String key, Instant updated, Optional<String> assignee, Status status) { + if (assignee == null || assignee.isPresent() && assignee.get().isEmpty()) // TODO: Throw on these things + assignee = Optional.empty(); + this.id = id; + this.key = key; + this.updated = updated; + this.assignee = assignee; + this.status = status; + } + + public IssueInfo withAssignee(Optional<String> assignee) { + return new IssueInfo(id, key, updated, assignee, status); + } + + public String id() { return id; } + public String key() { return key; } + public Instant updated() { return updated; } + public Optional<String> assignee() { return assignee; } + public Status status() { return status; } + + public enum Status { + + toDo("To Do"), + inProgress("In Progress"), + done("Done"), + noCategory("No Category"); + + private final String value; + + Status(String value) { this.value = value; } + + public static Status fromValue(String value) { + for (Status status : Status.values()) + if (status.value.equals(value)) + return status; + throw new IllegalArgumentException(value + " is not a valid status."); + } + + } + + } + + + /** + * A representation of an issue with a Vespa application which can be reported and escalated through an external issue service. + * This class is immutable. + * + * @author jvenstad + */ + class Issue { + + private final String summary; + private final String description; + private final Classification classification; + + public Issue(String summary, String description, Classification classification) { + if (summary.isEmpty()) throw new IllegalArgumentException("Summary can not be empty."); + if (description.isEmpty()) throw new IllegalArgumentException("Description can not be empty."); + + this.summary = summary; + this.description = description; + this.classification = classification; + } + + public Issue(String summary, String description) { + this(summary, description, null); + } + + public Issue with(Classification classification) { + return new Issue(summary, description, classification); + } + public Issue withDescription(String description) { return new Issue(summary, description, classification); } + + /** Return a new @Issue with the description of @this, but with @appendage appended. */ + public Issue append(String appendage) { + return new Issue(summary, description + "\n\n" + appendage, classification); + } + + public String summary() { return summary; } + public String description() { return description; } + public Optional<Classification> classification() { return Optional.ofNullable(classification); } + + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java new file mode 100644 index 00000000000..2068bc7e92d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; + +/** + * A service which returns metric values on request + * + * @author bratseth + */ +public interface MetricsService { + + ApplicationMetrics getApplicationMetrics(ApplicationId application); + + DeploymentMetrics getDeploymentMetrics(ApplicationId application, Zone zone); + + class DeploymentMetrics { + + private final double queriesPerSecond; + private final double writesPerSecond; + private final long documentCount; + private final double queryLatencyMillis; + private final double writeLatencyMillis; + + public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, + long documentCount, + double queryLatencyMillis, double writeLatencyMillis) { + this.queriesPerSecond = queriesPerSecond; + this.writesPerSecond = writesPerSecond; + this.documentCount = documentCount; + this.queryLatencyMillis = queryLatencyMillis; + this.writeLatencyMillis = writeLatencyMillis; + } + + public double queriesPerSecond() { return queriesPerSecond; } + + public double writesPerSecond() { return writesPerSecond; } + + public long documentCount() { return documentCount; } + + public double queryLatencyMillis() { return queryLatencyMillis; } + + public double writeLatencyMillis() { return writeLatencyMillis; } + + } + + class ApplicationMetrics { + + private final double queryServiceQuality; + private final double writeServiceQuality; + + public ApplicationMetrics(double queryServiceQuality, double writeServiceQuality) { + this.queryServiceQuality = queryServiceQuality; + this.writeServiceQuality = writeServiceQuality; + } + + /** Returns the quality of service for queries as a number between 1 (perfect) and 0 (none) */ + public double queryServiceQuality() { return queryServiceQuality; } + + /** Returns the quality of service for writes as a number between 1 (perfect) and 0 (none) */ + public double writeServiceQuality() { return writeServiceQuality; } + + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java new file mode 100644 index 00000000000..652b5495bc5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import java.util.Optional; + +/** + * @author jvenstad + */ +public interface Properties { + + /** + * Return the @Issues.Classification listed for the property with id @propertyId. + */ + Optional<Issues.Classification> classificationFor(long propertyId); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java new file mode 100644 index 00000000000..cb5731164c8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +/** + * @author mpolden + */ +public enum ApplicationAction { + deploy("deployer"), + read("reader"), + write("writer"); + + public final String roleName; + + ApplicationAction(String roleName) { + this.roleName = roleName; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java new file mode 100644 index 00000000000..c1f72fa4370 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +/** + * Interface for integrating controller with Athens. + * + * @author mpolden + */ +public interface Athens { + + String principalTokenHeader(); + AthensPrincipal principalFrom(ScrewdriverId screwdriverId); + AthensPrincipal principalFrom(UserId userId); + NTokenValidator validator(); + NToken nTokenFrom(String rawToken); + UnauthorizedZmsClient unauthorizedZmsClient(); + ZmsClientFactory zmsClientFactory(); + AthensDomain screwdriverDomain(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java new file mode 100644 index 00000000000..58b878870b9 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.security.Principal; +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthensPrincipal implements Principal { + + private final AthensDomain domain; + private final UserId userId; + + public AthensPrincipal(AthensDomain domain, UserId userId) { + this.domain = domain; + this.userId = userId; + } + + public UserId getUserId() { + return userId; + } + + public AthensDomain getDomain() { + return domain; + } + + public String toYRN() { + return domain.id() + "." + userId.id(); + } + + @Override + public String toString() { + return toYRN(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthensPrincipal that = (AthensPrincipal) o; + return Objects.equals(domain, that.domain) && + Objects.equals(userId, that.userId); + } + + @Override + public int hashCode() { + return Objects.hash(domain, userId); + } + + @Override + public String getName() { + return userId.id(); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java new file mode 100644 index 00000000000..9bbb5f28d8f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import java.security.PublicKey; +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthensPublicKey { + private final PublicKey publicKey; + private final String keyId; + + public AthensPublicKey(PublicKey publicKey, String keyId) { + this.publicKey = publicKey; + this.keyId = keyId; + } + + public PublicKey getPublicKey() { + return publicKey; + } + + public String getKeyId() { + return keyId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthensPublicKey that = (AthensPublicKey) o; + return Objects.equals(publicKey, that.publicKey) && + Objects.equals(keyId, that.keyId); + } + + @Override + public int hashCode() { + return Objects.hash(publicKey, keyId); + } + + @Override + public String toString() { + return "AthensPublicKey{" + + "publicKey=" + publicKey + + ", keyId='" + keyId + '\'' + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java new file mode 100644 index 00000000000..42af966be3d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java @@ -0,0 +1,51 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthensService { + + private final AthensDomain domain; + private final String serviceName; + + public AthensService(AthensDomain domain, String serviceName) { + this.domain = domain; + this.serviceName = serviceName; + } + + public String toFullServiceName() { + return domain.id() + "." + serviceName; + } + + public AthensDomain getDomain() { + return domain; + } + + public String getServiceName() { + return serviceName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthensService that = (AthensService) o; + return Objects.equals(domain, that.domain) && + Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(domain, serviceName); + } + + @Override + public String toString() { + return String.format("AthensService(%s)", toFullServiceName()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java new file mode 100644 index 00000000000..9c21d5814cb --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +/** + * @author bjorncs + */ +public class InvalidTokenException extends Exception { + public InvalidTokenException(String message) { + super(message); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java new file mode 100644 index 00000000000..b74872b4c6a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.security.PublicKey; + +/** + * @author mpolden + */ +public interface NToken { + + AthensPrincipal getPrincipal(); + UserId getUser(); + AthensDomain getDomain(); + String getToken(); + String getKeyId(); + void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java new file mode 100644 index 00000000000..905d7d864a3 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +/** + * @author mpolden + */ +public interface NTokenValidator { + + void preloadPublicKeys(); + AthensPrincipal validate(NToken nToken) throws InvalidTokenException; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java new file mode 100644 index 00000000000..d1996bdbd45 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.List; + +/** + * @author gv + */ +public class UnauthorizedZmsClient { + + private final ZmsClient client; + + public UnauthorizedZmsClient(ZmsClientFactory zmsClientFactory) { + client = zmsClientFactory.createClientWithoutPrincipal(); + } + + public List<AthensDomain> getDomainList(String prefix) { + return client.getDomainList(prefix); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java new file mode 100644 index 00000000000..7ff54957e16 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.List; + +/** + * @author bjorncs + */ +public interface ZmsClient { + void createTenant(AthensDomain tenantDomain); + + void deleteTenant(AthensDomain tenantDomain); + + void addApplication(AthensDomain tenantDomain, ApplicationId applicationName); + + void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName); + + boolean hasApplicationAccess(AthensPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName); + + boolean hasTenantAdminAccess(AthensPrincipal principal, AthensDomain tenantDomain); + + // Used before vespa tenancy is established for the domain. + boolean isDomainAdmin(AthensPrincipal principal, AthensDomain domain); + + List<AthensDomain> getDomainList(String prefix); + + List<AthensDomain> getTenantDomainsForUser(AthensPrincipal principal); + + AthensPublicKey getPublicKey(AthensService service, String keyId); + + List<AthensPublicKey> getPublicKeys(AthensService service); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java new file mode 100644 index 00000000000..24a2d67ebf6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +/** + * @author bjorncs + */ +public interface ZmsClientFactory { + ZmsClient createClientWithServicePrincipal(); + + ZmsClient createClientWithAuthorizedServiceToken(NToken authorizedServiceToken); + + ZmsClient createClientWithoutPrincipal(); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java new file mode 100644 index 00000000000..ed5b2daca86 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +/** + * @author bjorncs + */ +public class ZmsException extends RuntimeException { + + private final int code; + + public ZmsException(Throwable t, int code) { + super(t.getMessage(), t); + this.code = code; + } + + public ZmsException(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java new file mode 100644 index 00000000000..4f8e5f5ff05 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java @@ -0,0 +1,19 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import java.security.PublicKey; +import java.util.Optional; + +/** + * Interface for a keystore containing public keys for Athens services + * + * @author bjorncs + */ +@FunctionalInterface +public interface ZmsKeystore { + Optional<PublicKey> getPublicKey(AthensService service, String keyId); + + default void preloadKeys(AthensService service) { + // Default implementation is noop + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java new file mode 100644 index 00000000000..8a02d0dcff5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens.mock; + +import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * @author bjorncs + */ +public class AthensDbMock { + + public final Map<AthensDomain, Domain> domains = new HashMap<>(); + + public AthensDbMock addDomain(Domain domain) { + domains.put(domain.name, domain); + return this; + } + + public static class Domain { + + public final AthensDomain name; + public final Set<AthensPrincipal> admins = new HashSet<>(); + public final Set<AthensPrincipal> tenantAdmins = new HashSet<>(); + public final Map<ApplicationId, Application> applications = new HashMap<>(); + public boolean isVespaTenant = false; + + public Domain(AthensDomain name) { + this.name = name; + } + + public Domain admin(AthensPrincipal user) { + admins.add(user); + return this; + } + + public Domain tenantAdmin(AthensPrincipal user) { + tenantAdmins.add(user); + return this; + } + + /** + * Simulates establishing Vespa tenancy in Athens. + */ + public void markAsVespaTenant() { + isVespaTenant = true; + } + + } + + public static class Application { + + public final Map<ApplicationAction, Set<AthensPrincipal>> acl = new HashMap<>(); + + public Application() { + acl.put(ApplicationAction.deploy, new HashSet<>()); + acl.put(ApplicationAction.read, new HashSet<>()); + acl.put(ApplicationAction.write, new HashSet<>()); + } + + public Application addRoleMember(ApplicationAction action, AthensPrincipal user) { + acl.get(action).add(user); + return this; + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java new file mode 100644 index 00000000000..a993c6e3da3 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java @@ -0,0 +1,95 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens.mock; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NTokenValidator; +import com.yahoo.vespa.hosted.controller.api.integration.athens.UnauthorizedZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; + +/** + * @author mpolden + */ +public class AthensMock extends AbstractComponent implements Athens { + + private static final AthensDomain userDomain = new AthensDomain("domain1"); + private static final AthensDomain screwdriverDomain = new AthensDomain("screwdriver-domain"); + + private final ZmsClientFactory zmsClientFactory; + private final UnauthorizedZmsClient unauthorizedZmsClient; + private final NTokenValidator nTokenValidator; + + public AthensMock(AthensDbMock athensDb, NTokenValidator nTokenValidator) { + this.zmsClientFactory = new ZmsClientFactoryMock(athensDb); + this.unauthorizedZmsClient = new UnauthorizedZmsClient(zmsClientFactory); + this.nTokenValidator = nTokenValidator; + } + + public AthensMock(AthensDbMock athensDbMock) { + this(athensDbMock, mockValidator); + } + + @Inject + public AthensMock() { + this(new AthensDbMock(), mockValidator); + } + + @Override + public String principalTokenHeader() { + return "X-Athens-Token"; + } + + @Override + public AthensPrincipal principalFrom(ScrewdriverId screwdriverId) { + return new AthensPrincipal(screwdriverDomain, new UserId("screwdriver-" + screwdriverId.id())); + } + + @Override + public AthensPrincipal principalFrom(UserId userId) { + return new AthensPrincipal(userDomain, userId); + } + + @Override + public NTokenValidator validator() { + return nTokenValidator; + } + + @Override + public NToken nTokenFrom(String rawToken) { + return new NTokenMock(rawToken); + } + + @Override + public UnauthorizedZmsClient unauthorizedZmsClient() { + return unauthorizedZmsClient; + } + + @Override + public ZmsClientFactory zmsClientFactory() { + return zmsClientFactory; + } + + @Override + public AthensDomain screwdriverDomain() { + return screwdriverDomain; + } + + private static final NTokenValidator mockValidator = new NTokenValidator() { + @Override + public void preloadPublicKeys() { + } + + @Override + public AthensPrincipal validate(NToken nToken) throws InvalidTokenException { + return nToken.getPrincipal(); + } + }; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java new file mode 100644 index 00000000000..ae23a69e409 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java @@ -0,0 +1,68 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens.mock; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; + +import java.security.PublicKey; +import java.util.Objects; + +/** + * @author mpolden + */ +public class NTokenMock implements NToken { + + private static final AthensDomain domain = new AthensDomain("test"); + private static final UserId userId = new UserId("user"); + + private final String rawToken; + + public NTokenMock(String rawToken) { + this.rawToken = rawToken; + } + + @Override + public AthensPrincipal getPrincipal() { + return new AthensPrincipal(domain, userId); + } + + @Override + public UserId getUser() { + return userId; + } + + @Override + public AthensDomain getDomain() { + return domain; + } + + @Override + public String getToken() { + return "test-token"; + } + + @Override + public String getKeyId() { + return "test-key"; + } + + @Override + public void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException { + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NTokenMock)) return false; + NTokenMock that = (NTokenMock) o; + return Objects.equals(rawToken, that.rawToken); + } + + @Override + public int hashCode() { + return Objects.hash(rawToken); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java new file mode 100644 index 00000000000..73d971a27fe --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens.mock; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; + +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +public class ZmsClientFactoryMock extends AbstractComponent implements ZmsClientFactory { + + private static final Logger log = Logger.getLogger(ZmsClientFactoryMock.class.getName()); + + private final AthensDbMock athens; + + public ZmsClientFactoryMock() { + this(new AthensDbMock()); + } + + ZmsClientFactoryMock(AthensDbMock athens) { + this.athens = athens; + } + + public AthensDbMock getSetup() { + return athens; + } + + @Override + public ZmsClient createClientWithServicePrincipal() { + log("createClientWithServicePrincipal()"); + return new ZmsClientMock(athens); + } + + @Override + public ZmsClient createClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { + log("createClientWithAuthorizedServiceToken(authorizedServiceToken='%s')", authorizedServiceToken); + return new ZmsClientMock(athens); + } + + @Override + public ZmsClient createClientWithoutPrincipal() { + log("createClientWithoutPrincipal()"); + return new ZmsClientMock(athens); + } + + private static void log(String format, Object... args) { + log.log(Level.INFO, String.format(format, args)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java new file mode 100644 index 00000000000..97f391f792d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java @@ -0,0 +1,131 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.athens.mock; + +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensService; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +public class ZmsClientMock implements ZmsClient { + + private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName()); + + private final AthensDbMock athens; + + public ZmsClientMock(AthensDbMock athens) { + this.athens = athens; + } + + @Override + public void createTenant(AthensDomain tenantDomain) { + log("createTenant(tenantDomain='%s')", tenantDomain); + getDomainOrThrow(tenantDomain, false).isVespaTenant = true; + } + + @Override + public void deleteTenant(AthensDomain tenantDomain) { + log("deleteTenant(tenantDomain='%s')", tenantDomain); + AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, false); + domain.isVespaTenant = false; + domain.applications.clear(); + domain.tenantAdmins.clear(); + } + + @Override + public void addApplication(AthensDomain tenantDomain, ApplicationId applicationName) { + log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName); + AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); + if (!domain.applications.containsKey(applicationName)) { + domain.applications.put(applicationName, new AthensDbMock.Application()); + } + } + + @Override + public void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName) { + log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName); + getDomainOrThrow(tenantDomain, true).applications.remove(applicationName); + } + + @Override + public boolean hasApplicationAccess(AthensPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName) { + log("hasApplicationAccess(principal='%s', action='%s', tenantDomain='%s', applicationName='%s')", + principal, action, tenantDomain, applicationName); + AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); + AthensDbMock.Application application = domain.applications.get(applicationName); + if (application == null) { + throw zmsException(400, "Application '%s' not found", applicationName); + } + return domain.admins.contains(principal) || application.acl.get(action).contains(principal); + } + + @Override + public boolean hasTenantAdminAccess(AthensPrincipal principal, AthensDomain tenantDomain) { + log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", principal, tenantDomain); + return isDomainAdmin(principal, tenantDomain) || + getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(principal); + } + + @Override + public boolean isDomainAdmin(AthensPrincipal principal, AthensDomain domain) { + log("isDomainAdmin(principal='%s', domain='%s')", principal, domain); + return getDomainOrThrow(domain, false).admins.contains(principal); + } + + @Override + public List<AthensDomain> getDomainList(String prefix) { + log("getDomainList()"); + return new ArrayList<>(athens.domains.keySet()); + } + + @Override + public List<AthensDomain> getTenantDomainsForUser(AthensPrincipal principal) { + log("getTenantDomainsForUser(principal='%s')", principal); + return athens.domains.values().stream() + .filter(domain -> domain.tenantAdmins.contains(principal) || domain.admins.contains(principal)) + .map(domain -> domain.name) + .collect(toList()); + } + + @Override + public AthensPublicKey getPublicKey(AthensService service, String keyId) { + throw new UnsupportedOperationException(); + } + + @Override + public List<AthensPublicKey> getPublicKeys(AthensService service) { + throw new UnsupportedOperationException(); + } + + private AthensDbMock.Domain getDomainOrThrow(AthensDomain domainName, boolean verifyVespaTenant) { + AthensDbMock.Domain domain = Optional.ofNullable(athens.domains.get(domainName)) + .orElseThrow(() -> zmsException(400, "Domain '%s' not found", domainName)); + if (verifyVespaTenant && !domain.isVespaTenant) { + throw zmsException(400, "Domain not a Vespa tenant: '%s'", domainName); + } + return domain; + } + + private static ZmsException zmsException(int code, String message, Object... args) { + return new ZmsException(new RuntimeException(String.format(message, args)), code); + } + + private static void log(String format, Object... args) { + log.log(Level.INFO, String.format(format, args)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java new file mode 100644 index 00000000000..d4454503786 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.athens.mock; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java new file mode 100644 index 00000000000..eabe214abf2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.athens; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java new file mode 100644 index 00000000000..87970458855 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author mortent + */ +public class AttributeMapping { + + private final String attribute; + private final List<String> chefPath; + + private AttributeMapping(String attribute, List<String> chefPath) { + this.chefPath = chefPath; + this.attribute = attribute; + } + + public static AttributeMapping simpleMapping(String attribute) { + return new AttributeMapping(attribute, Collections.singletonList(attribute)); + } + + public static AttributeMapping deepMapping(String attribute, List<String> chefPath) { + return new AttributeMapping(attribute, chefPath); + } + + public String toString() { + return String.format("\"%s\": [%s]", attribute, + chefPath.stream().map(s -> String.format("\"%s\"", s)) + .collect(Collectors.joining(",")) + ); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java new file mode 100644 index 00000000000..693947b6f61 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef; + + +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; + +import java.net.URL; +import java.util.List; + +public interface Chef { + + ChefResource getApi(); + + ChefNode getNode(String name); + + Client getClient(String name); + + ChefNode deleteNode(String name); + + Client deleteClient(String name); + + NodeResult searchNodeByFQDN(String fqdn); + + NodeResult searchNodes(String query); + + PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> attributeMappings); + + void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName); + + ChefEnvironment getChefEnvironment(String environmentName); + + CookBook getCookbook(String cookbookName, String cookbookVersion); + + String downloadResource(URL resourceURL); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java new file mode 100644 index 00000000000..1b2dad34b8d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java @@ -0,0 +1,112 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef; + +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; + +import javax.ws.rs.NotFoundException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author mpolden + */ +public class ChefMock implements Chef { + + private final NodeResult result; + private final List<String> chefEnvironments; + + public ChefMock() { + result = new NodeResult(); + result.rows = new ArrayList<>(); + chefEnvironments = new ArrayList<>(); + chefEnvironments.add("hosted-verified-prod"); + chefEnvironments.add("hosted-infra-cd"); + } + + @Override + public ChefResource getApi() { + return null; + } + + @Override + public ChefNode getNode(String name) { + return null; + } + + @Override + public Client getClient(String name) { + return null; + } + + @Override + public ChefNode deleteNode(String name) { + return null; + } + + @Override + public Client deleteClient(String name) { + return null; + } + + public void addSearchResult(ChefNode node) { + result.rows.add(node); + } + + @Override + public NodeResult searchNodeByFQDN(String fqdn) { + return result; + } + + @Override + public NodeResult searchNodes(String query) { + return result; + } + + @Override + public PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> returnAttributes) { + PartialNodeResult partialNodeResult = new PartialNodeResult(); + partialNodeResult.rows = result.rows.stream() + .map(chefNode -> { + Map<String, String> data = new HashMap<>(); + data.put("fqdn", chefNode.name); + return new PartialNode(data); + }) + .collect(Collectors.toList()); + return partialNodeResult; + } + + @Override + public void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName) { + if(!chefEnvironments.contains(fromEnvironmentName)) { + throw new NotFoundException(String.format("Source chef environment %s does not exist", fromEnvironmentName)); + } + chefEnvironments.add(toEnvironmentName); + } + + @Override + public ChefEnvironment getChefEnvironment(String environmentName) { + return null; + } + + @Override + public CookBook getCookbook(String cookbookName, String cookbookVersion) { + return null; + } + + @Override + public String downloadResource(URL resourceURL) { + return ""; + } +} + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java new file mode 100644 index 00000000000..5d3d4b87b74 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.chef; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java new file mode 100644 index 00000000000..8576949280b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java @@ -0,0 +1,110 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ChefEnvironment { + + @JsonProperty("name") + private String name; + + @JsonProperty("default_attributes") + private Map<String, Object> attributes; + @JsonProperty("override_attributes") + private Map<String, Object> overrideAttributes; + @JsonProperty("description") + private String description; + @JsonProperty("cookbook_versions") + private Map<String, String> cookbookVersions; + + // internal + @JsonProperty("json_class") + private final String _jsonClass = "Chef::Environment"; + @JsonProperty("chef_type") + private final String _chefType = "environment"; + + public static Builder builder() { + return new Builder(); + } + + public String getName() { + return name; + } + + public Builder copy() { + return builder() + .name(name) + .attributes(attributes) + .overrideAttributes(overrideAttributes) + .cookbookVersions(cookbookVersions) + .description(description); + } + + public String getDescription() { + return description; + } + + public Map<String, String> getCookbookVersions() { + return cookbookVersions; + } + + public Map<String, Object> getAttributes() { + return attributes; + } + + public Map<String, Object> getOverrideAttributes() { + return overrideAttributes; + } + + public static class Builder { + private String name; + private Map<String, Object> attributes; + private String description; + private Map<String, Object> overrideAttributes; + private Map<String, String> cookbookVersions; + + public Builder name(String name){ + this.name = name; + return this; + } + + public Builder attributes(Map<String, Object> defaultAttributes) { + this.attributes = defaultAttributes; + return this; + } + + public Builder overrideAttributes(Map<String, Object> overrideAttributes) { + this.overrideAttributes = overrideAttributes; + return this; + } + + public Builder cookbookVersions(Map<String, String> cookbookVersions) { + this.cookbookVersions = cookbookVersions; + return this; + } + + public Builder description(String description) { + this.description = description; + return this; + } + + public ChefEnvironment build() { + ChefEnvironment chefEnvironment = new ChefEnvironment(); + chefEnvironment.name = name; + chefEnvironment.description = description; + chefEnvironment.cookbookVersions = cookbookVersions; + chefEnvironment.attributes = attributes; + chefEnvironment.overrideAttributes = overrideAttributes; + + return chefEnvironment; + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java new file mode 100644 index 00000000000..08d9a1045e8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java @@ -0,0 +1,118 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ChefNode { + + @JsonProperty("name") + public String name; + + @JsonProperty("chef_environment") + public String chefEnvironment; + + @JsonProperty("run_list") + public List<String> runList; + + @JsonProperty("json_class") + public String jsonClass; + + @JsonProperty("chef_type") + public String chefType; + + @JsonProperty("automatic") + public Map<String, Object> automaticAttributes; + + @JsonProperty("normal") + public Map<String, Object> normalAttributes; + + @JsonProperty("default") + public Map<String, Object> defaultAttributes; + + @JsonProperty("override") + public Map<String, Object> overrideAttributes; + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(ChefNode src) { + return new Builder(src); + } + + public static class Builder { + private String name; + private String chefEnvironment; + private List<String> runList; + private String jsonClass; + private String chefType; + private Map<String, Object> automaticAttributes; + private Map<String, Object> normalAttributes; + private Map<String, Object> defaultAttributes; + private Map<String, Object> overrideAttributes; + + private Builder(){} + + private Builder(ChefNode src){ + this.name = src.name; + this.chefEnvironment = src.chefEnvironment; + this.runList = new ArrayList<>(src.runList); + this.jsonClass = src.jsonClass; + this.chefType = src.chefType; + this.automaticAttributes = new HashMap<>(src.automaticAttributes); + this.normalAttributes = new HashMap<>(src.normalAttributes); + this.defaultAttributes = new HashMap<>(src.defaultAttributes); + this.overrideAttributes = new HashMap<>(src.overrideAttributes); + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder chefEnvironment(String chefEnvironment) { + this.chefEnvironment = chefEnvironment; + return this; + } + + public ChefNode build(){ + ChefNode node = new ChefNode(); + node.name = this.name; + node.chefEnvironment = this.chefEnvironment; + node.runList = this.runList; + node.jsonClass = this.jsonClass; + node.chefType = this.chefType; + node.automaticAttributes = this.automaticAttributes; + node.overrideAttributes = this.overrideAttributes; + node.defaultAttributes = this.defaultAttributes; + node.normalAttributes = this.normalAttributes; + return node; + } + + } + + @Override + public String toString() { + return "Node{" + + "name='" + name + '\'' + + ", chefEnvironment='" + chefEnvironment + '\'' + + ", runList=" + runList + + ", jsonClass='" + jsonClass + '\'' + + ", chefType='" + chefType + '\'' + + ", automaticAttributes=" + automaticAttributes + + ", normalAttributes=" + normalAttributes + + ", defaultAttributes=" + defaultAttributes + + ", overrideAttributes=" + overrideAttributes + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java new file mode 100644 index 00000000000..98eeb0770fc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java @@ -0,0 +1,74 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import java.util.List; + +/** + * @author mortent + * @author mpolden + */ + +@Path("/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public interface ChefResource { + + @Path("/organizations/{organization}/environments/{environment}/nodes") + @Consumes("application/json") + @GET + List<String> getNodes(@PathParam("organization") String organization, @PathParam("environment") String environment); + + @GET + @Path("/organizations/{organization}/nodes/{nodename}") + ChefNode getNode(@PathParam("organization") String organization, @PathParam("nodename") String nodename); + + @PUT + @Path("/organizations/{organization}/nodes/{nodename}") + ChefNode updateNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName, String node); + + @DELETE + @Path("/organizations/{organization}/nodes/{nodename}") + ChefNode deleteNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName); + + @GET + @Path("/organizations/{organization}/clients/{name}") + Client getClient(@PathParam("organization") String organization, @PathParam("name") String name); + + @DELETE + @Path("/organizations/{organization}/clients/{name}") + Client deleteClient(@PathParam("organization") String organization, @PathParam("name") String name); + + @GET + @Path("/organizations/{organization}/environments/{environment}") + ChefEnvironment getEnvironment(@PathParam("organization") String organization, @PathParam("environment") String environment); + + @PUT + @Path("/organizations/{organization}/environments/{name}") + String updateEnvironment(@PathParam("organization") String organization, @PathParam("name") String chefEnvironmentName, String contentAsString); + + @POST + @Path("/organizations/{organization}/environments") + String createEnvironment(@PathParam("organization") String organization, String contentAsString); + + @GET + @Path("/organizations/{organization}/search/node") + NodeResult searchNode(@PathParam("organization") String organization, @QueryParam("q") String query); + + @POST + @Path("/organizations/{organization}/search/node") + PartialNodeResult partialSearchNode(@PathParam("organization") String organization, @QueryParam("q") String query, @QueryParam("rows") int rows, String keys); + + @GET + @Path("/organizations/{organization}/cookbooks/{name}/{version}") + CookBook getCookBook(@PathParam("organization") String organization, @PathParam("name") String name, @PathParam("version") String version); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java new file mode 100644 index 00000000000..0ea9b0e9997 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java @@ -0,0 +1,25 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Client { + + @JsonProperty("name") + public String name; + @JsonProperty("validator") + public boolean validator; + + @Override + public String toString() { + return "Client{" + + "name='" + name + '\'' + + ", validator=" + validator + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java new file mode 100644 index 00000000000..ab49ac9ff60 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CookBook { + public final String name; + public final List<Attributes> attributes; + + public CookBook(@JsonProperty("name") String name, @JsonProperty("attributes") List<Attributes> attributes) { + this.name = name; + this.attributes = attributes; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Attributes { + public final String name; + public final String url; + + public Attributes(@JsonProperty("name") String name, @JsonProperty("url") String url) { + this.name = name; + this.url = url; + } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java new file mode 100644 index 00000000000..e3ab431473f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class NodeResult { + @JsonProperty("total") + public int total; + @JsonProperty("start") + public int start; + @JsonProperty("rows") + public List<ChefNode> rows; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java new file mode 100644 index 00000000000..f4aa90021b1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; +import java.util.Optional; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class PartialNode { + + @JsonProperty("data") + private final Map<String, String> data; + + @JsonCreator + public PartialNode(@JsonProperty("data") Map<String, String> data) { + this.data = data; + } + + public Optional<String> getValue(String key) { + return Optional.ofNullable(data.get(key)); + } + + public String getFqdn() { + return getValue("fqdn").orElse(""); + } + + public String getName() { + return getValue("name").orElse(""); + } + + public Double getOhaiTime() { + return Double.parseDouble(getValue("ohai_time").orElse("0.0")); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java new file mode 100644 index 00000000000..9925237a193 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class PartialNodeResult { + @JsonProperty("total") + public int total; + @JsonProperty("start") + public int start; + @JsonProperty("rows") + public List<PartialNode> rows; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java new file mode 100644 index 00000000000..7d06571507e --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.chef.rest; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java new file mode 100644 index 00000000000..1958c5bd0ff --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.fasterxml.jackson.databind.JsonNode; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * @author Oyvind Grønnesby + */ +public interface ConfigServerClient { + + interface PreparedApplication { + void activate(); + List<Log> messages(); + PrepareResponse prepareResponse(); + } + + PreparedApplication prepare(DeploymentId applicationInstance, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content); + + List<String> getNodeQueryHost(DeploymentId applicationInstance, String type) throws NoInstanceException; + + void restart(DeploymentId applicationInstance, Optional<Hostname> hostname) throws NoInstanceException; + + void deactivate(DeploymentId applicationInstance) throws NoInstanceException; + + JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds); + + JsonNode grabLog(DeploymentId applicationInstance); + + ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region); + + Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); + + /** Returns the version this particular config server is running */ + Version version(URI configserverUri); + + /** + * Set new status on en endpoint in one zone. + * + * @param deployment The application/zone pair + * @param endpoint The endpoint to modify + * @param status The new status with metadata + * @throws IOException If trouble contacting the server + */ + void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status) throws IOException; + + /** + * Get the endpoint status for an app in one zone + * + * @param deployment The application/zone pair + * @param endpoint The endpoint to modify + * @return The endpoint status with metadata + * @throws IOException If trouble contacting the server + */ + EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) throws IOException; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java new file mode 100644 index 00000000000..f578322ac76 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -0,0 +1,41 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import java.net.URI; + +/** + * @author Tony Vaagenes + */ +public class ConfigServerException extends RuntimeException { + + private final URI serverUri; + private final ErrorCode errorCode; + + public ConfigServerException(URI serverUri, String message, ErrorCode errorCode, Throwable cause) { + super(message, cause); + this.serverUri = serverUri; + this.errorCode = errorCode; + } + + public ErrorCode getErrorCode() { + return errorCode; + } + + public URI getServerUri() { + return serverUri; + } + + // TODO: Copied from Vespa. Expose these in Vespa and use them here + public enum ErrorCode { + APPLICATION_LOCK_FAILURE, + BAD_REQUEST, + INTERNAL_SERVER_ERROR, + INVALID_APPLICATION_PACKAGE, + METHOD_NOT_ALLOWED, + NOT_FOUND, + OUT_OF_CAPACITY, + REQUEST_TIMEOUT, + UNKNOWN_VESPA_VERSION + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java new file mode 100644 index 00000000000..ba5d740d0e1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Tony Vaagenes + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Log { + public long time; + public String level; + public String message; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java new file mode 100644 index 00000000000..a415721407f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +/** + * @author Tony Vaagenes + */ +public class NoInstanceException extends Exception { + public NoInstanceException(String msg) { + super(msg); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java new file mode 100644 index 00000000000..6054e05149b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; + +import java.net.URI; +import java.util.List; + +/** + * @author Tony Vaagenes + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class PrepareResponse { + public TenantId tenant; + @JsonProperty("activate") public URI activationUri; + public String message; + public List<Log> log; + public ConfigChangeActions configChangeActions; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java new file mode 100644 index 00000000000..10eddb2628f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java new file mode 100644 index 00000000000..9bc9cfa8ed0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java @@ -0,0 +1,105 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import java.util.HashMap; +import java.util.Map; + +/** + * Cost data model for an application instance. I.e one running vespa application in one zone. + * + * @author smorgrav + */ +// TODO: Make immutable +// TODO: Make the Application own this and rename to Cost +// TODO: Enforce constraints +// TODO: Remove application id elements +// TODO: Model zone as Zone +// TODO: Cost per zone + total +// TODO: Use doubles +public class ApplicationCost { + + /** This contains environment.region */ + private String zone; + + private String tenant; + + // This must contain applicationName.instanceName. TODO: Fix + private String app; + + private int tco; + private float utilization; + private float waste; + Map<String, ClusterCost> cluster; + + /** Create an empty (invalid) application cost */ + public ApplicationCost() {} + + public ApplicationCost(String zone, String tenant, String app, int tco, float utilization, float waste, + Map<String, ClusterCost> clusterCost) { + this.zone = zone; + this.tenant = tenant; + this.app = app; + this.tco = tco; + this.utilization = utilization; + this.waste = waste; + cluster = new HashMap<>(clusterCost); + } + + public String getZone() { + return zone; + } + + public void setZone(String zone) { + this.zone = zone; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public Map<String, ClusterCost> getCluster() { + return cluster; + } + + public void setCluster(Map<String, ClusterCost> cluster) { + this.cluster = cluster; + } + + public int getTco() { + return tco; + } + + public void setTco(int tco) { + if (tco < 0) throw new IllegalArgumentException("TCO cannot be negative"); + this.tco = tco; + } + + public String getTenant() { + return tenant; + } + + public void setTenant(String tenant) { + this.tenant = tenant; + } + + public float getUtilization() { + return utilization; + } + + public void setUtilization(float utilization) { + if (utilization < 0) throw new IllegalArgumentException("Utilization cannot be negative"); + this.utilization = utilization; + } + + public float getWaste() { + return waste; + } + + public void setWaste(float waste) { + this.waste = waste; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java new file mode 100644 index 00000000000..d9edf22d42c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; + +import java.util.List; + +/** + * Interface for retrieving cost data directly or indirectly from yamas and + * the noderepository. + * + * + * @author smorgrav + */ +public interface Backend { + List<ApplicationCost> getApplicationCost(); + ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId appId) throws NotFoundCheckedException; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java new file mode 100644 index 00000000000..1e41325a4fd --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java @@ -0,0 +1,182 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import java.util.List; + +/** + * Cost data model for a cluster. I.e one cluster within one vespa application in one zone. + * + * @author smorgrav + */ +// TODO: Use doubles +// TODO: Make immutable +// TODO: Enforce constraints +// TODO: Document content +public class ClusterCost { + + private int count; + private String resource; + private float utilization; + private int tco; + private String flavor; + private int waste; + private String type; + private float utilMem; + private float utilCpu; + private float utilDisk; + private float utilDiskBusy; + private float usageMem; + private float usageCpu; + private float usageDisk; + private float usageDiskBusy; + private List<String> hostnames; + + /** Create an empty (invalid) cluster cost */ + public ClusterCost() {} + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } + + public String getFlavor() { + return flavor; + } + + public void setFlavor(String flavor) { + this.flavor = flavor; + } + + public List<String> getHostnames() { + return hostnames; + } + + public void setHostnames(List<String> hostnames) { + this.hostnames = hostnames; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } + + public int getTco() { + return tco; + } + + public void setTco(int tco) { + this.tco = tco; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public float getUtilization() { + return utilization; + } + + public void setUtilization(float utilization) { + validateUtilRatio(utilization); + this.utilization = utilization; + } + + public int getWaste() { + return waste; + } + + public void setWaste(int waste) { + this.waste = waste; + } + + public float getUsageCpu() { + return usageCpu; + } + + public void setUsageCpu(float usageCpu) { + validateUsageRatio(usageCpu); + this.usageCpu = usageCpu; + } + + public float getUsageDisk() { + return usageDisk; + } + + public void setUsageDisk(float usageDisk) { + validateUsageRatio(usageDisk); + this.usageDisk = usageDisk; + } + + public float getUsageMem() { + return usageMem; + } + + public void setUsageMem(float usageMem) { + validateUsageRatio(usageMem); + this.usageMem = usageMem; + } + + public float getUtilCpu() { + return utilCpu; + } + + public void setUtilCpu(float utilCpu) { + validateUtilRatio(utilCpu); + this.utilCpu = utilCpu; + } + + public float getUtilDisk() { + return utilDisk; + } + + public void setUtilDisk(float utilDisk) { + validateUtilRatio(utilDisk); + this.utilDisk = utilDisk; + } + + public float getUtilMem() { + return utilMem; + } + + public void setUtilMem(float utilMem) { + validateUsageRatio(utilMem); + this.utilMem = utilMem; + } + + public float getUsageDiskBusy() { + return usageDiskBusy; + } + + public void setUsageDiskBusy(float usageDiskBusy) { + validateUsageRatio(usageDiskBusy); + this.usageDiskBusy = usageDiskBusy; + } + + public float getUtilDiskBusy() { + return utilDiskBusy; + } + + public void setUtilDiskBusy(float utilDiskBusy) { + validateUtilRatio(utilDiskBusy); + this.utilDiskBusy = utilDiskBusy; + } + + private void validateUsageRatio(float ratio) { + if (ratio < 0) throw new IllegalArgumentException("Usage cannot be negative"); + if (ratio > 1) throw new IllegalArgumentException("Usage exceed 1 (using more than it has available)"); + } + + private void validateUtilRatio(float ratio) { + if (ratio < 0) throw new IllegalArgumentException("Utilization cannot be negative"); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java new file mode 100644 index 00000000000..7297b60de5c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java @@ -0,0 +1,53 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; + +import java.util.List; + +/** + * Cost domain model declaration + * + * @author smorgrav + */ +public interface Cost { + + /** + * Calculate a list of the applications that is wasting most + * in absolute terms. To improve utilization, it should make + * sense to focus on this list. + * + * @return An ordered set of applications with the highest potential for + * improved CPU utilization across all environments and regions. + */ + List<ApplicationCost> getCPUAnalysis(int nofApplications); + + /** + * Collect all information and format it as a Cvs blob for download. + * + * @return A String with comma separated values. Can be big! + */ + String getCsvForLocalAnalysis(); + + /** + * Get application costs for all applications across all regions and environments + * + * @return A list of applications in given zone + */ + List<ApplicationCost> getApplicationCost(); + + /** + * Get application costs for a given application instance in a given zone. + * + * @param env Environment like test, dev, perf, staging or prod + * @param region Region name like us-east-1 + * @param app ApplicationId like tenant:application:instance + * + * @return A list of applications in given zone + */ + ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId app) + throws NotFoundCheckedException; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java new file mode 100644 index 00000000000..088b1fa12bc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java @@ -0,0 +1,93 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.hosted.controller.api.cost.CostJsonModel; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +/** + * Converting from cost data model to the JSON data model used in the cost REST API. + * + * @author smorgrav + */ +public class CostJsonModelAdapter { + + public static CostJsonModel.Application toJsonModel(ApplicationCost appCost) { + CostJsonModel.Application app = new CostJsonModel.Application(); + app.zone = appCost.getZone(); + app.tenant = appCost.getTenant(); + app.app = appCost.getApp(); + app.tco = appCost.getTco(); + app.utilization = appCost.getUtilization(); + app.waste = appCost.getWaste(); + app.cluster = new HashMap<>(); + Map<String, ClusterCost> clusterMap = appCost.getCluster(); + for (String key : clusterMap.keySet()) { + app.cluster.put(key, toJsonModel(clusterMap.get(key))); + } + + return app; + } + + public static void toSlime(ApplicationCost appCost, Cursor object) { + object.setString("zone", appCost.getZone()); + object.setString("tenant", appCost.getTenant()); + object.setString("app", appCost.getApp()); + object.setLong("tco", appCost.getTco()); + object.setDouble("utilization", appCost.getUtilization()); + object.setDouble("waste", appCost.getWaste()); + Cursor clustersObject = object.setObject("cluster"); + for (Map.Entry<String, ClusterCost> clusterEntry : appCost.getCluster().entrySet()) + toSlime(clusterEntry.getValue(), clustersObject.setObject(clusterEntry.getKey())); + } + + public static CostJsonModel.Cluster toJsonModel(ClusterCost clusterCost) { + CostJsonModel.Cluster cluster = new CostJsonModel.Cluster(); + cluster.count = clusterCost.getCount(); + cluster.resource = clusterCost.getResource(); + cluster.utilization = clusterCost.getUtilization(); + cluster.tco = clusterCost.getTco(); + cluster.flavor = clusterCost.getFlavor(); + cluster.waste = clusterCost.getWaste(); + cluster.type = clusterCost.getType(); + cluster.util = new CostJsonModel.HardwareResources(); + cluster.util.cpu = clusterCost.getUtilCpu(); + cluster.util.mem = clusterCost.getUtilMem(); + cluster.util.disk = clusterCost.getUtilDisk(); + cluster.usage = new CostJsonModel.HardwareResources(); + cluster.usage.cpu = clusterCost.getUsageCpu(); + cluster.usage.mem = clusterCost.getUsageMem(); + cluster.usage.disk = clusterCost.getUsageDisk(); + cluster.hostnames = new ArrayList<>(clusterCost.getHostnames()); + cluster.usage.diskBusy = clusterCost.getUsageDiskBusy(); + cluster.util.diskBusy = clusterCost.getUtilDiskBusy(); + return cluster; + } + + private static void toSlime(ClusterCost clusterCost, Cursor object) { + object.setLong("count", clusterCost.getCount()); + object.setString("resource", clusterCost.getResource()); + object.setDouble("utilization", clusterCost.getUtilization()); + object.setLong("tco", clusterCost.getTco()); + object.setString("flavor", clusterCost.getFlavor()); + object.setLong("waste", clusterCost.getWaste()); + object.setString("type", clusterCost.getType()); + Cursor utilObject = object.setObject("util"); + utilObject.setDouble("cpu", clusterCost.getUtilCpu()); + utilObject.setDouble("mem", clusterCost.getUtilMem()); + utilObject.setDouble("disk", clusterCost.getUtilDisk()); + utilObject.setDouble("diskBusy", clusterCost.getUtilDiskBusy()); + Cursor usageObject = object.setObject("usage"); + usageObject.setDouble("cpu", clusterCost.getUsageCpu()); + usageObject.setDouble("mem", clusterCost.getUsageMem()); + usageObject.setDouble("disk", clusterCost.getUsageDisk()); + usageObject.setDouble("diskBusy", clusterCost.getUsageDiskBusy()); + Cursor hostnamesArray = object.setArray("hostnames"); + for (String hostname : clusterCost.getHostnames()) + hostnamesArray.addString(hostname); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java new file mode 100644 index 00000000000..f08e6cc9b36 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.cost; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java new file mode 100644 index 00000000000..f70afb3a0a0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +/** + * An in-memory name service for testing purposes. + * + * @author mpolden + */ +public class MemoryNameService implements NameService { + + private final Set<Record> records = new HashSet<>(); + + @Override + public RecordId createCname(String alias, String canonicalName) { + records.add(new Record("CNAME", alias, canonicalName)); + return new RecordId(UUID.randomUUID().toString()); + } + + @Override + public Optional<Record> findRecord(Record.Type type, String name) { + return records.stream() + .filter(record -> record.type() == type && record.name().equals(name)) + .findFirst(); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java new file mode 100644 index 00000000000..2ccce23b60c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import java.util.Optional; + +/** + * A managed DNS service. + * + * @author mpolden + */ +public interface NameService { + + /** + * Create a new CNAME record + * + * @param alias The alias to create + * @param canonicalName The canonical name which the alias should point to. This must be a domain. + */ + RecordId createCname(String alias, String canonicalName); + + /** Find record by type and name */ + Optional<Record> findRecord(Record.Type type, String name); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java new file mode 100644 index 00000000000..0782a82da79 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java @@ -0,0 +1,74 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import java.util.Objects; + +/** + * A basic representation of a DNS resource record, containing only the record type, name and value. + * + * @author mpolden + */ +public class Record { + + private final Type type; + private final String name; + private final String value; + + public Record(Type type, String name, String value) { + this.type = type; + this.name = name; + this.value = value; + } + + public Record(String type, String name, String value) { + this(Type.valueOf(type), name, value); + } + + public Type type() { + return type; + } + + public String value() { + return value; + } + + public String name() { + return name; + } + + public enum Type { + A, + AAAA, + CNAME, + MX, + NS, + PTR, + SOA, + SRV, + TXT + } + + @Override + public String toString() { + return "Record{" + + "type=" + type + + ", name='" + name + '\'' + + ", value='" + value + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Record)) return false; + Record record = (Record) o; + return type == record.type && + Objects.equals(name, record.name); + } + + @Override + public int hashCode() { + return Objects.hash(type, name); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java new file mode 100644 index 00000000000..9c47be12855 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +/** + * Unique identifier for a resource record. + * + * @author mpolden + */ +public class RecordId { + + private final String id; + + public RecordId(String id) { + this.id = id; + } + + public String id() { + return id; + } + + @Override + public String toString() { + return "RecordId{" + + "id='" + id + '\'' + + '}'; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java new file mode 100644 index 00000000000..e075b544ce8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.dns; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java new file mode 100644 index 00000000000..fc242a360f6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.entity; + +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.util.Map; +import java.util.Set; + +/** + * A service which provides access to business-specific entities. + * + * @author mpolden + */ +public interface EntityService { + + /** List all properties known by the service */ + Map<PropertyId, Property> listProperties(); + + /** List all groups of which user is a member */ + Set<UserGroup> getUserGroups(UserId user); + + /** Whether user is a member of the group */ + boolean isGroupMember(UserId user, UserGroup group); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java new file mode 100644 index 00000000000..e5c2bbedae4 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java @@ -0,0 +1,37 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.entity; + +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * @author mpolden + */ +public class MemoryEntityService implements EntityService { + + @Override + public Map<PropertyId, Property> listProperties() { + Map<PropertyId, Property> properties = new HashMap<>(); + properties.put(new PropertyId("1234"), new Property("foo")); + properties.put(new PropertyId("4321"), new Property("bar")); + return Collections.unmodifiableMap(properties); + } + + @Override + public Set<UserGroup> getUserGroups(UserId userId) { + return Collections.singleton(new UserGroup("vespa")); + } + + @Override + public boolean isGroupMember(UserId userId, UserGroup userGroup) { + return true; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java new file mode 100644 index 00000000000..1e74f4ca372 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.entity; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java new file mode 100644 index 00000000000..1cb3f73441b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.github; + +/** + * @author mpolden + */ +public interface GitHub { + + GitSha getCommit(String owner, String repo, String ref); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java new file mode 100644 index 00000000000..9a398ef7cb5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.github; + +import java.time.Instant; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * @author jvenstad + */ +public class GitHubMock implements GitHub { + + private final Map<String, GitSha> tags = new HashMap<>(); + private boolean mockAny = true; + + @Override + public GitSha getCommit(String owner, String repo, String ref) { + if (mockAny) { + String sha = UUID.randomUUID().toString(); + return new GitSha(sha, new GitSha.GitCommit(new GitSha.GitAuthor("foo", "foo@foo.tld", + Date.from(Instant.EPOCH)))); + } + if (tags.containsKey(ref)) { + return tags.get(ref); + } + throw new IllegalArgumentException("Unknown ref: " + ref); + } + + public GitHubMock knownTag(String tag, String sha) { + this.tags.put(tag, new GitSha(sha, new GitSha.GitCommit( + new GitSha.GitAuthor("foo", "foo@foo.tld", Date.from(Instant.EPOCH))))); + return this; + } + + public GitHubMock mockAny(boolean mockAny) { + this.mockAny = mockAny; + return this; + } + + public GitHubMock reset() { + tags.clear(); + mockAny = true; + return this; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java new file mode 100644 index 00000000000..4aac98f1708 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.github; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Date; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class GitSha { + + @JsonProperty("sha") + public final String sha; + + @JsonProperty("commit") + public final GitCommit commit; + + @JsonCreator + public GitSha(@JsonProperty("sha") String sha, @JsonProperty("commit") GitCommit commit) { + this.sha = sha; + this.commit = commit; + } + + public static class GitCommit { + @JsonProperty("author") + public final GitAuthor author; + + @JsonCreator + public GitCommit(@JsonProperty("author") GitAuthor author) { + this.author = author; + } + } + + public static class GitAuthor { + + @JsonProperty("name") + public final String name; + @JsonProperty("email") + public final String email; + @JsonProperty("date") + public final Date date; + + @JsonCreator + public GitAuthor(@JsonProperty("name") String name, @JsonProperty("email") String email, + @JsonProperty("date") Date date) { + this.name = name; + this.email = email; + this.date = date; + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java new file mode 100644 index 00000000000..ec20c05c374 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.github; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java new file mode 100644 index 00000000000..30bf23f18a9 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import java.util.List; + +/** + * @author mortent + */ +public interface Jira { + + List<JiraIssue> searchByProjectAndSummary(String project, String summary); + + JiraIssue createIssue(JiraCreateIssue issue); + + void commentIssue(JiraIssue issue, JiraComment comment); + + void addAttachment(JiraIssue issue, String filename, String fileContent); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java new file mode 100644 index 00000000000..2d67b720fe0 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class JiraComment { + + public final String body; + + @JsonCreator + public JiraComment(@JsonProperty("body") String body) { + this.body = body; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java new file mode 100644 index 00000000000..e5e35af4475 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java @@ -0,0 +1,86 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class JiraCreateIssue { + + @JsonProperty("fields") + public final JiraFields fields; + + public JiraCreateIssue(JiraFields fields) { + this.fields = fields; + } + + public static class JiraFields { + @JsonProperty("summary") + public final String summary; + + @JsonProperty("description") + public final String description; + + @JsonProperty("project") + public final JiraProject project; + + @JsonProperty("issuetype") + public final JiraIssueType issueType; + + @JsonProperty("components") + public final List<JiraComponent> components; + + public JiraFields( + JiraProject project, + String summary, + String description, + JiraIssueType issueType, + List<JiraComponent> components) { + this.project = project; + this.summary = summary; + this.description = description; + this.issueType = issueType; + this.components = components; + } + + + public static class JiraProject { + public static final JiraProject VESPA = new JiraProject("VESPA"); + + @JsonProperty("key") + public final String key; + + public JiraProject(String key) { + this.key = key; + } + } + + public static class JiraIssueType { + public static final JiraIssueType DEFECT = new JiraIssueType("Defect"); + + @JsonProperty("name") + public final String name; + + public JiraIssueType(String name) { + this.name = name; + } + } + + public static class JiraComponent { + public static final JiraComponent COREDUMPS = new JiraComponent("CoreDumps"); + + @JsonProperty("name") + public final String name; + + + public JiraComponent(String name) { + this.name = name; + } + } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java new file mode 100644 index 00000000000..d88e75d3a58 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java @@ -0,0 +1,45 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.Date; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class JiraIssue { + public final String key; + private final Fields fields; + + @JsonCreator + public JiraIssue(@JsonProperty("key") String key, @JsonProperty("fields") Fields fields) { + this.key = key; + this.fields = fields; + } + + public Instant lastUpdated() { + return fields.lastUpdated; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Fields { + final Instant lastUpdated; + + @JsonCreator + public Fields( + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'hh:mm:ss.SSSZ", timezone = "UTC") + @JsonProperty("updated") Date updated) { + lastUpdated = updated.toInstant(); + } + + public Fields(Instant instant) { + this.lastUpdated = instant; + } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java new file mode 100644 index 00000000000..809ac8360bb --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.List; + +/** + * @author mortent + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class JiraIssues { + public final List<JiraIssue> issues; + + @JsonCreator + public JiraIssues(@JsonProperty("issues") List<JiraIssue> issues) { + this.issues = issues == null ? Collections.emptyList() : issues; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java new file mode 100644 index 00000000000..da653ddd8a8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author jvenstad + */ +// TODO: Make mock. +public class JiraMock implements Jira { + + public final Map<String, JiraCreateIssue.JiraFields> issues = new HashMap<>(); + + private Long counter = 0L; + + @Override + public List<JiraIssue> searchByProjectAndSummary(String project, String summary) { + return issues.entrySet().stream() + .filter(entry -> entry.getValue().project.key.equals(project)) + .filter(entry -> entry.getValue().summary.contains(summary)) + .map(entry -> new JiraIssue(entry.getKey(), new JiraIssue.Fields(Instant.now()))) + .collect(Collectors.toList()); + } + + @Override + public JiraIssue createIssue(JiraCreateIssue issueData) { + JiraIssue issue = uniqueKey(); + issues.put(issue.key, issueData.fields); + return issue; + } + + @Override + public void commentIssue(JiraIssue issue, JiraComment comment) { + // Add mock when relevant. + } + + @Override + public void addAttachment(JiraIssue issue, String filename, String fileContent) { + // Add mock when relevant. + } + + private JiraIssue uniqueKey() { + return new JiraIssue((++counter).toString(), new JiraIssue.Fields(Instant.now())); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java new file mode 100644 index 00000000000..efe356c69e9 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.jira; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java new file mode 100644 index 00000000000..265d57cadd8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java new file mode 100644 index 00000000000..d49d6a9e4c2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.routing; + +import java.util.Map; + +/** + * A global routing service. + * + * @author mpolden + */ +public interface GlobalRoutingService { + + /** Returns the health status for each endpoint behind the given rotation name */ + Map<String, RotationStatus> getHealthStatus(String rotationName); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java new file mode 100644 index 00000000000..9f1ac1b1f0b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.routing; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * @author bratseth + */ +public class MemoryGlobalRoutingService implements GlobalRoutingService { + + @Override + public Map<String, RotationStatus> getHealthStatus(String rotationName) { + HashMap<String, RotationStatus> map = new HashMap<>(); + map.put("prod.us-west-1", RotationStatus.IN); + return Collections.unmodifiableMap(map); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java new file mode 100644 index 00000000000..8c59bb44fa1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.routing; + +/** + * Represents the health status of a global rotation. + * + * @author andreer + */ +public enum RotationStatus { + IN, OUT, UNKNOWN +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java new file mode 100644 index 00000000000..a4bf733bd2c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java @@ -0,0 +1,30 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.routing; + +/** + * @author smorgrav + */ +public class RoutingEndpoint { + + private final boolean isGlobal; + private final String endpoint; + + public RoutingEndpoint(String endpoint, boolean isGlobal) { + this.endpoint = endpoint; + this.isGlobal = isGlobal; + } + + /** + * @return True if the endpoint is global + */ + public boolean isGlobal() { + return isGlobal; + } + + /* + * @return The URI for the endpoint + */ + public String getEndpoint() { + return endpoint; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java new file mode 100644 index 00000000000..276e19da8f6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java @@ -0,0 +1,19 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.routing; + +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; + +import java.util.List; + +/** + * @author bratseth + * @author smorgrav + */ +public interface RoutingGenerator { + + /** + * @param deploymentId Specifying an application in a zone + * @return List of endpoints for that deploymentId + */ + List<RoutingEndpoint> endpoints(DeploymentId deploymentId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java new file mode 100644 index 00000000000..25374003ec1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.routing; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java new file mode 100644 index 00000000000..98c664eb07d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.security; + +/** + * A service for retrieving secrets, such as API keys, private keys and passwords. + * + * @author mpolden + */ +public interface KeyService { + + String getSecret(String key); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java new file mode 100644 index 00000000000..296eebf8ea5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.security; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java new file mode 100644 index 00000000000..9114cf20ccc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.vespa.hosted.controller.api.integration.Contacts; + +import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author mpolden + */ +public class ContactsMock implements Contacts { + + private final Map<Long, List<UserContact>> userContacts = new HashMap<>(); + + public void addContact(long propertyId, List<UserContact> contacts) { + userContacts.put(propertyId, contacts); + } + + public List<UserContact> userContactsFor(long propertyId) { + return userContacts.get(propertyId); + } + + @Override + public URI contactsUri(long propertyId) { + return URI.create("http://contacts.test?propertyId=" + propertyId); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java new file mode 100644 index 00000000000..160f80076bd --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java @@ -0,0 +1,87 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.vespa.hosted.controller.api.integration.Issues; + +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +/** + * An memory backed implementation of the Issues API which logs changes and does nothing else. + * + * @author bratseth + */ +public class LoggingIssues implements Issues { + + private static final Logger log = Logger.getLogger(LoggingIssues.class.getName()); + + /** Used to fabricate unique issue ids */ + private AtomicLong issueIdSequence = new AtomicLong(0); + + // These two maps should have precisely the same keys + private final Map<String, Issue> issues = new HashMap<>(); + private final Map<String, IssueInfo> issueInfos = new HashMap<>(); + + @Override + public IssueInfo fetch(String issueId) { + return issueInfos.getOrDefault(issueId, + new IssueInfo(issueId, null, Instant.ofEpochMilli(0), null, IssueInfo.Status.noCategory)); + } + + @Override + public List<IssueInfo> fetchSimilarTo(Issue issue) { + return Collections.emptyList(); + } + + @Override + public String file(Issue issue) { + log.info("Want to file " + issue); + String issueId = "issue-" + issueIdSequence.getAndIncrement(); + issues.put(issueId, issue); + issueInfos.put(issueId, new IssueInfo(issueId, null, Instant.now(), null, IssueInfo.Status.noCategory)); + return issueId; + } + + @Override + public void update(String issueId, String description) { + log.info("Want to update " + issueId); + issues.put(issueId, requireIssue(issueId).withDescription(description)); + } + + @Override + public void reassign(String issueId, String assignee) { + log.info("Want to reassign issue " + issueId + " to " + assignee); + issueInfos.put(issueId, requireInfo(issueId).withAssignee(Optional.of(assignee))); + } + + @Override + public void addWatcher(String issueId, String watcher) { + log.info("Want to add watcher " + watcher + " to issue " + issueId); + } + + @Override + public void comment(String issueId, String comment) { + log.info("Want to comment on issue " + issueId); + } + + private Issue requireIssue(String issueId) { + Issue issue = issues.get(issueId); + if (issue == null) + throw new IllegalArgumentException("No issue with id '" + issueId + "'"); + return issue; + } + + private IssueInfo requireInfo(String issueId) { + IssueInfo info = issueInfos.get(issueId); + if (info == null) + throw new IllegalArgumentException("No issue info with id '" + issueId + "'"); + return info; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java new file mode 100644 index 00000000000..53a31933e03 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.vespa.hosted.controller.api.integration.Issues; +import com.yahoo.vespa.hosted.controller.api.integration.Properties; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * @author mpolden + */ +public class PropertiesMock implements Properties { + + private final Map<Long, Issues.Classification> projects = new HashMap<>(); + + public void addClassification(long propertyId, String classification) { + projects.put(propertyId, new Issues.Classification(classification)); + } + + public Optional<Issues.Classification> classificationFor(long propertyId) { + return Optional.ofNullable(projects.get(propertyId)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java new file mode 100644 index 00000000000..2aab38dc66d --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * No-dependency implementations of integration interfaces for setups where we want to avoid contacting + * certain thirds-party systems. + * + * @author bratseth + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java new file mode 100644 index 00000000000..e7bdf786c8c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; + +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +/** + * Provides information about zones in a hosted Vespa system. + * + * @author mpolden + */ +public interface ZoneRegistry { + + SystemName system(); + List<Zone> zones(); + Optional<Zone> getZone(Environment environment, RegionName region); + List<URI> getConfigServerUris(Environment environment, RegionName region); + Optional<URI> getLogServerUri(Environment environment, RegionName region); + Optional<Duration> getDeploymentTimeToLive(Environment environment, RegionName region); + URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application); + URI getDashboardUri(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java new file mode 100644 index 00000000000..148564a373f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.zone; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java new file mode 100644 index 00000000000..78a6750aedb --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java @@ -0,0 +1,14 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.nonpublic; + +/** + * Non public header fields that are not part of the public api. + * + * Placed here since this is the only module we own that both the + * command-line client and controller-server depend on. + * + * @author Tony Vaagenes + */ +public class HeaderFields { + public static final String USER_ID_HEADER_FIELD = "vespa.hosted.trusted.username"; +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java new file mode 100644 index 00000000000..ed3e69bcac7 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java @@ -0,0 +1,39 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.rotation; + +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; + +import java.util.Objects; + +/** + * Represents a global routing rotation. + * + * @author Oyvind Gronnesby + */ +public class Rotation { + + /** The ID of the allocated rotation. This value is generated by global routing system. */ + public final RotationId rotationId; + + /** The global name which the allocated rotation points to */ + public final String rotationName; + + public Rotation(RotationId rotationId, String rotationName) { + this.rotationId = Objects.requireNonNull(rotationId); + this.rotationName = Objects.requireNonNull(rotationName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Rotation)) return false; + final Rotation rotation = (Rotation) o; + return rotationId.equals(rotation.rotationId) && rotationName.equals(rotation.rotationName); + } + + @Override + public int hashCode() { + return Objects.hash(rotationId, rotationName); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java new file mode 100644 index 00000000000..1626158a489 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.rotation; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java new file mode 100644 index 00000000000..65c5e0f9365 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.statuspage; + +import com.fasterxml.jackson.databind.JsonNode; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + +/** + * @author andreer + */ +@Path("/v1/") +@Produces(MediaType.APPLICATION_JSON) +public interface StatusPageResource { + + @GET + @Path("{page}") + @Produces(MediaType.APPLICATION_JSON) + JsonNode statusPage(@PathParam("page") String page, @QueryParam("since") String since); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java new file mode 100644 index 00000000000..3f9117bf931 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.statuspage; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java new file mode 100644 index 00000000000..7bb4bfc6467 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.zone.v1; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.List; + +/** + * Used by build system and command-line tool. + * + * @author smorgrav + */ +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Path(ZoneApi.API_VERSION) +public interface ZoneApi { + + String API_VERSION = "v1"; + + @GET + @Path("") + List<ZoneReference.Environment> listEnvironments(); + + @GET + @Path("/environment/{environment}") + List<ZoneReference.Region> listRegions(@PathParam("environment") String env); + + @GET + @Path("/environment/{environment}/default") + ZoneReference.Region defaultRegion(@PathParam("environment") String env); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java new file mode 100644 index 00000000000..82d03d72acd --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.zone.v1; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.net.URI; + +/** + * @author smorgrav + */ +public class ZoneReference { + + public static class Environment { + @JsonProperty("name") + private String name; + + @JsonProperty("url") + private URI url; + + public String getName() { + return name; + } + + public Environment setName(String name) { + this.name = name; + return this; + } + + public URI getUrl() { + return url; + } + + public Environment setUrl(URI url) { + this.url = url; + return this; + } + } + + public static class Region { + @JsonProperty("name") + private String name; + + @JsonProperty("url") + private URI url; + + public String getName() { + return name; + } + + public Region setName(String name) { + this.name = name; + return this; + } + + public URI getUrl() { + return url; + } + + public Region setUrl(URI url) { + this.url = url; + return this; + } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java new file mode 100644 index 00000000000..e3275ff35fa --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.zone.v1; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java new file mode 100644 index 00000000000..97d99e262b5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java @@ -0,0 +1,110 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.zone.v2; + +import com.fasterxml.jackson.databind.JsonNode; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; +import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +/** + * Aka the controller proxy service. + * + * Proxies calls to correct config server with the additional feature of + * retry and fail detection (ping). + */ +@Path(ZoneApiV2.API_VERSION) +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public interface ZoneApiV2 { + + String API_VERSION = "v2"; + + @GET + @Path("/") + ZoneReferences listZones(); + + @GET + @Path("/{environment}/{region}/{proxy_request: .+}") + Response proxyGet( + @PathParam("environment") String env, + @PathParam("region") String region, + @PathParam("proxy_request") String proxyRequest, + @Context HttpServletRequest request); + + @POST + @Path("/{environment}/{region}/{proxy_request: .+}") + Response proxyPost( + @PathParam("environment") String env, + @PathParam("region") String region, + @PathParam("proxy_request") String proxyRequest, + @Context HttpServletRequest request); + + @PUT + @Path("/{environment}/{region}/{proxy_request: .+}") + Response proxyPut( + @PathParam("environment") String env, + @PathParam("region") String region, + @PathParam("proxy_request") String proxyRequest, + @Context HttpServletRequest request); + + @DELETE + @Path("/{environment}/{region}/{proxy_request: .+}") + Response proxyDelete( + @PathParam("environment") String env, + @PathParam("region") String region, + @PathParam("proxy_request") String proxyRequest, + @Context HttpServletRequest request); + + // Explicit mappings of some proxy requests (to enable creation of proxy clients with javax.ws.rs) + + @GET + @Path("/{environmentId}/{regionId}/application/v2/tenant/{tenantId}/application/{applicationId}/environment/{environmentId}/region/{regionId}/instance/{instanceId}/serviceconverge") + Response waitForConfigConvergeV2(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId, + @QueryParam("timeout") long timeoutInSeconds); + @GET + @Path("/{environmentId}/{regionId}/application/v2/tenant/{tenantId}/application/{applicationId}/environment/{environmentId}/region/{regionId}/instance/{instanceId}/serviceconverge/{host}") + Response waitForConfigConvergeV2(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("instanceId") InstanceId instanceId, + @PathParam("host") String host, + @QueryParam("timeout") long timeoutInSeconds); + + @GET + @Path("/{environmentId}/{regionId}/config/v2/tenant/{tenantId}/application/{applicationId}/prelude.fastsearch.documentdb-info/{clusterid}/search/cluster.{clusterid}") + JsonNode getConfigWithDocumentTypes(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @PathParam("clusterid") String clusterid, + @QueryParam("timeout") long timeoutInSeconds); + + @GET + @Path("/{environmentId}/{regionId}/config/v2/tenant/{tenantId}/application/{applicationId}/cloud.config.cluster-list") + JsonNode getVespaConfigClusterList(@PathParam("tenantId") TenantId tenantId, + @PathParam("applicationId") ApplicationId applicationId, + @PathParam("environmentId") EnvironmentId environmentId, + @PathParam("regionId") RegionId regionId, + @QueryParam("timeout") long timeoutInSeconds); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java new file mode 100644 index 00000000000..95dc1c2ee7c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.zone.v2; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.hosted.controller.api.configserver.Environment; +import com.yahoo.vespa.hosted.controller.api.configserver.Region; + +/** + * @author mpolden + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ZoneReference { + + @JsonProperty("environment") + public final Environment environment; + @JsonProperty("region") + public final Region region; + + @JsonCreator + public ZoneReference(@JsonProperty("environment") Environment environment, @JsonProperty("region") Region region) { + this.environment = environment; + this.region = region; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java new file mode 100644 index 00000000000..3a219afa0a6 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.zone.v2; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.List; + +/** + * Wire format for listing the controller URIs for all the available zones + * + * @author smorgrav + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ZoneReferences { + + @JsonProperty("uris") + public final List<String> uris; + + @JsonProperty("zones") + public final List<ZoneReference> zones; + + @JsonCreator + public ZoneReferences(@JsonProperty("uris") List<String> uris, @JsonProperty("zones") List<ZoneReference> zones) { + this.uris = Collections.unmodifiableList(uris); + this.zones = Collections.unmodifiableList(zones); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java new file mode 100644 index 00000000000..5d4b1310981 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.zone.v2; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java new file mode 100644 index 00000000000..1cdff0f920b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.common; + +/** + * Constants for request context attributes used in our APIs. + * + * @author mpolden + */ +public interface ContextAttributes { + + String SECURITY_CONTEXT_ATTRIBUTE = "vespa.hosted.security_context"; + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java new file mode 100644 index 00000000000..a55a7e2bdfc --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.common; + +/** + * We have tons of places where we throw exceptions when + * some hosted resource is not found. This is usually + * done with IllegalArgumentExceptions, java.ws.rs exceptions or + * the servermodel runtime exceptions in the controller-server module. + * + * This is a checked alternative to do the same thing. + * + * @author smorgrav + */ +public class NotFoundCheckedException extends Exception { + + public NotFoundCheckedException() { + super(); + } + + public NotFoundCheckedException(String msg) { + super(msg); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java new file mode 100644 index 00000000000..95decd86e8b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.common; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java new file mode 100644 index 00000000000..4a02fe23dec --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import org.junit.Test; + +import java.io.IOException; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class DeployOptionsTest { + + @Test + public void it_serializes_version() throws IOException { + DeployOptions options = new DeployOptions(Optional.empty(), Optional.of(new Version("6.98.227")), false, false); + final ObjectMapper objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .registerModule(new Jdk8Module()); + + String string = objectMapper.writeValueAsString(options); + assertEquals("{\"screwdriverBuildJob\":null,\"vespaVersion\":\"6.98.227\",\"ignoreValidationErrors\":false,\"deployCurrentVersion\":false}", string); + } +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java new file mode 100644 index 00000000000..56825cf7c61 --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java @@ -0,0 +1,153 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.identifiers; + +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class IdentifierTest { + + @Test(expected = IllegalArgumentException.class) + public void existing_tenant_id_not_empty() { + new TenantId(""); + } + + @Test(expected = IllegalArgumentException.class) + public void existing_tenant_id_must_check_pattern() { + new TenantId("`"); + } + + @Test(expected = IllegalArgumentException.class) + public void default_not_allowed_for_tenants() { + new TenantId("default"); + } + + @Test + public void existing_tenant_id_must_accept_valid_id() { + new TenantId("msbe"); + } + + @Test(expected = IllegalArgumentException.class) + public void existing_tenant_id_cannot_be_uppercase() { + new TenantId("MixedCaseTenant"); + } + + @Test(expected = IllegalArgumentException.class) + public void existing_tenant_id_cannot_contain_dots() { + new TenantId("tenant.with.dots"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_tenant_id_cannot_contain_underscore() { + TenantId.validate("underscore_tenant"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_tenant_id_cannot_contain_dot() { + TenantId.validate("tenant.with.dots"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_tenant_id_cannot_contain_uppercase() { + TenantId.validate("UppercaseTenant"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_tenant_id_cannot_start_with_dash() { + TenantId.validate("-tenant"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_tenant_id_cannot_end_with_dash() { + TenantId.validate("tenant-"); + } + + @Test(expected = IllegalArgumentException.class) + public void existing_application_id_cannot_be_uppercase() { + new ApplicationId("MixedCaseApplication"); + } + + @Test(expected = IllegalArgumentException.class) + public void existing_application_id_cannot_contain_dots() { + new ApplicationId("application.with.dots"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_application_id_cannot_contain_underscore() { + ApplicationId.validate("underscore_application"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_application_id_cannot_contain_dot() { + ApplicationId.validate("application.with.dots"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_application_id_cannot_contain_uppercase() { + ApplicationId.validate("UppercaseApplication"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_application_id_cannot_start_with_dash() { + ApplicationId.validate("-application"); + } + + @Test(expected = IllegalArgumentException.class) + public void new_application_id_cannot_end_with_dash() { + ApplicationId.validate("application-"); + } + + @Test(expected = IllegalArgumentException.class) + public void instance_id_cannot_be_uppercase() { + new InstanceId("MixedCaseInstance"); + } + + @Test + public void rotation_id_may_contain_dot() { + new RotationId("rotation.id.with.dot"); + } + + @Test + public void user_tenant_id_does_not_contain_underscore() { + assertEquals("by-under-score-user", new UserId("under_score_user").toTenantId().id()); + } + + @Test + public void athens_parent_domain_is_without_name_suffix() { + assertEquals(new AthensDomain("yby.john"), new AthensDomain("yby.john.myapp").getParent()); + } + + @Test + public void athens_domain_name_is_last_suffix() { + assertEquals("myapp", new AthensDomain("yby.john.myapp").getName()); + } + + @Test + public void domain_without_dot_is_toplevel() { + assertTrue(new AthensDomain("toplevel").isTopLevelDomain()); + assertFalse(new AthensDomain("not.toplevel").isTopLevelDomain()); + } + + @Test + public void dns_names_has_no_underscore() { + assertEquals("a-b-c", new ApplicationId("a_b_c").toDns()); + } + + @Test(expected = IllegalArgumentException.class) + public void identifiers_cannot_be_named_api() { + new ApplicationId("api"); + } + + + @Test + public void application_instance_id_dotted_string_is_subindentifers_concatinated_with_dots() { + DeploymentId id = new DeploymentId(com.yahoo.config.provision.ApplicationId.from("tenant", "application", "instance"), + new Zone(Environment.prod, RegionName.from("region"))); + assertEquals("tenant.application.prod.region.instance", id.dottedString()); + } +} diff --git a/controller-server/OWNERS b/controller-server/OWNERS new file mode 100644 index 00000000000..e6a0537ba53 --- /dev/null +++ b/controller-server/OWNERS @@ -0,0 +1,2 @@ +bratseth +mpolden diff --git a/controller-server/pom.xml b/controller-server/pom.xml new file mode 100644 index 00000000000..f9e84693452 --- /dev/null +++ b/controller-server/pom.xml @@ -0,0 +1,211 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>controller-server</artifactId> + <packaging>container-plugin</packaging> + <version>6-SNAPSHOT</version> + + <dependencies> + + <!-- provided --> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>controller-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-dev</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zkfacade</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-jersey2</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>serviceview</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <classifier>no_aop</classifier> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <scope>provided</scope> + </dependency> + + <!-- compile --> + + <dependency> + <groupId>commons-fileupload</groupId> + <artifactId>commons-fileupload</artifactId> + <version>1.3.1</version> + </dependency> + + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model-api</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.6</version> + </dependency> + + <!-- test --> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.easytesting</groupId> + <artifactId>fest-assert</artifactId> + <version>1.4</version> + <scope>test</scope> + </dependency> + + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <configuration> + <useCommonAssemblyIds>false</useCommonAssemblyIds> + <WebInfUrl>/WEB-INF/web.xml</WebInfUrl> + </configuration> + <extensions>true</extensions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Xlint:-deprecation</arg> + <arg>-Xlint:-try</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>target/${project.artifactId}-deploy.jar</file> + <type>jar</type> + <classifier>deploy</classifier> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java new file mode 100644 index 00000000000..ffe7cb6ef67 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; + +/** + * @author Tony Vaagenes + */ +public class AlreadyExistsException extends IllegalArgumentException { + + /** + * Example message: Tenant 'myId' already exists. + * + * @param capitalizedType e.g. Tenant, Application + * @param id The id of the entity that didn't exist. + * + */ + public AlreadyExistsException(String capitalizedType, String id) { + super(String.format("%s '%s' already exists", capitalizedType, id)); + } + + public AlreadyExistsException(Identifier identifier) { + this(identifier.capitalizedType(), identifier.id()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java new file mode 100644 index 00000000000..971438e008c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -0,0 +1,276 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; + +import java.time.Instant; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * An instance of an application. + * + * This is immutable. + * + * @author bratseth + */ +public class Application { + + private final ApplicationId id; + private final DeploymentSpec deploymentSpec; + private final ValidationOverrides validationOverrides; + private final Map<Zone, Deployment> deployments; + private final DeploymentJobs deploymentJobs; + private final Optional<Change> deploying; + private final boolean outstandingChange; + + /** Creates an empty application */ + public Application(ApplicationId id) { + this(id, DeploymentSpec.empty, ValidationOverrides.empty, ImmutableMap.of(), new DeploymentJobs(0L), + Optional.empty(), false); // TODO: Get rid of the 0 + } + + /** Used from persistence layer: Do not use */ + public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, + List<Deployment> deployments, + DeploymentJobs deploymentJobs, Optional<Change> deploying, boolean outstandingChange) { + this(id, deploymentSpec, validationOverrides, + deployments.stream().collect(Collectors.toMap(d -> d.zone(), d -> d)), + deploymentJobs, deploying, outstandingChange); + } + + private Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, + Map<Zone, Deployment> deployments, + DeploymentJobs deploymentJobs, Optional<Change> deploying, boolean outstandingChange) { + Objects.requireNonNull(id, "id cannot be null"); + Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); + Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null"); + Objects.requireNonNull(deployments, "deployments cannot be null"); + Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null"); + Objects.requireNonNull(deploying, "deploying cannot be null"); + this.id = id; + this.deploymentSpec = deploymentSpec; + this.validationOverrides = validationOverrides; + this.deployments = ImmutableMap.copyOf(deployments); + this.deploymentJobs = deploymentJobs; + this.deploying = deploying; + this.outstandingChange = outstandingChange; + } + + public ApplicationId id() { return id; } + + /** + * Returns the last deployed deployment spec of this application, + * or the empty deployment spec if it has never been deployed + */ + public DeploymentSpec deploymentSpec() { return deploymentSpec; } + + /** + * Returns the last deployed validation overrides of this application, + * or the empty validation overrides if it has never been deployed + * (or was deployed with an empty/missing validation overrides) + */ + public ValidationOverrides validationOverrides() { return validationOverrides; } + + /** Returns an immutable map of the current deployments of this */ + public Map<Zone, Deployment> deployments() { return deployments; } + + public DeploymentJobs deploymentJobs() { return deploymentJobs; } + + /** + * Returns the change that is currently in the process of being deployed on this application, + * or empty if no change is currently being deployed. + */ + public Optional<Change> deploying() { return deploying; } + + /** + * Returns whether this has an outstanding change (in the source repository), which + * has currently not started deploying (because a deployment is (or was) already in progress + */ + public boolean hasOutstandingChange() { return outstandingChange; } + + /** + * Returns the oldest version this has deployed in a permanent zone (not test or staging), + * or empty version if it is not deployed anywhere + */ + public Optional<Version> deployedVersion() { + return deployments().values().stream() + .filter(deployment -> isPermanent(deployment.zone().environment())) + .sorted(Comparator.comparing(Deployment::version)) + .findFirst() + .map(Deployment::version); + } + + /** The version that should be used to compile this application */ + public Version compileVersion(Controller controller) { + return deployedVersion().orElse(controller.systemVersion()); + } + + public Application withProjectId(long projectId) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), deploying, outstandingChange); + } + + public Application withJiraIssueId(Optional<String> jiraIssueId) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withJiraIssueId(jiraIssueId), deploying, outstandingChange); + } + + public Application withJobCompletion(JobReport report, Instant notificationTime, Controller controller) { + return new Application(id, + deploymentSpec, + validationOverrides, + deployments, + deploymentJobs.withCompletion(report, notificationTime, controller), + deploying, + outstandingChange); + } + + public Application withJobTriggering(JobType type, Instant triggerTime, Controller controller) { + return new Application(id, + deploymentSpec, + validationOverrides, + deployments, + deploymentJobs.withTriggering(type, + determineTriggerVersion(type, controller), + determineTriggerRevision(type, controller), + triggerTime), + deploying, + outstandingChange); + } + + public Application with(Deployment deployment) { + Map<Zone, Deployment> deployments = new LinkedHashMap<>(this.deployments); + deployments.put(deployment.zone(), deployment); + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application with(DeploymentJobs deploymentJobs) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application withoutDeploymentIn(Zone zone) { + Map<Zone, Deployment> deployments = new LinkedHashMap<>(this.deployments); + deployments.remove(zone); + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application withoutDeploymentJob(JobType jobType) { + DeploymentJobs deploymentJobs = this.deploymentJobs.without(jobType); + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application with(DeploymentSpec deploymentSpec) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application with(ValidationOverrides validationOverrides) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application withDeploying(Optional<Change> deploying) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + public Application withOutstandingChange(boolean outstandingChange) { + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange); + } + + private Version determineTriggerVersion(JobType jobType, Controller controller) { + Optional<Zone> zone = jobType.zone(controller.system()); + if ( ! zone.isPresent()) // a sloppy test TODO: Fix + return controller.systemVersion(); + return currentDeployVersion(controller, zone.get()); + } + + /** Returns the version a deployment to this zone should use for this application */ + Version currentDeployVersion(Controller controller, Zone zone) { + if ( ! deploying().isPresent()) + return currentVersion(controller, zone); + else if ( deploying().get() instanceof Change.ApplicationChange) + return currentVersion(controller, zone); + else + return ((Change.VersionChange) deploying().get()).version(); + } + + /** Returns the current version this application has, or if none; should use, in the given zone */ + Version currentVersion(Controller controller, Zone zone) { + Deployment currentDeployment = deployments().get(zone); + if (currentDeployment != null) // Already deployed in this zone: Use that version + return currentDeployment.version(); + + return deployedVersion().orElse(controller.systemVersion()); + } + + private Optional<ApplicationRevision> determineTriggerRevision(JobType jobType, Controller controller) { + Optional<Zone> zone = jobType.zone(controller.system()); + if ( ! zone.isPresent()) // a sloppy test TODO: Fix + return Optional.empty(); + return currentDeployRevision(jobType.zone(controller.system()).get()); + } + + /** Returns the version a deployment to this zone should use for this application, or empty if we don't know */ + Optional<ApplicationRevision> currentDeployRevision(Zone zone) { + if ( ! deploying().isPresent()) + return currentRevision(zone); + else if ( deploying().get() instanceof Change.VersionChange) + return currentRevision(zone); + else + return ((Change.ApplicationChange)deploying().get()).revision(); + } + + /** + * Returns the current revision this application has, or if none; should use assuming no change, + * in the given zone. Empty if not known + */ + Optional<ApplicationRevision> currentRevision(Zone zone) { + Deployment currentDeployment = deployments().get(zone); + if (currentDeployment != null) // Already deployed in this zone: Use that revision + return Optional.of(currentDeployment.revision()); + return Optional.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (! (o instanceof Application)) return false; + + Application that = (Application) o; + + return id.equals(that.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public String toString() { + return "application '" + id + "'"; + } + + private boolean isPermanent(Environment environment) { + if (environment == Environment.dev) return false; + if (environment == Environment.perf) return false; + if (environment == Environment.test) return false; + if (environment == Environment.staging) return false; + return true; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java new file mode 100644 index 00000000000..51bf530ed4a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -0,0 +1,541 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.api.ApplicationAlias; +import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; +import com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.rotation.RotationRepository; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A singleton owned by the Controller which contains the methods and state for controlling applications. + * + * @author bratseth + */ +public class ApplicationController { + + private static final Logger log = Logger.getLogger(ApplicationController.class.getName()); + + /** The controller owning this */ + private final Controller controller; + + /** For permanent storage */ + private final ControllerDb db; + /** For working memory storage and sharing between controllers */ + private final CuratorDb curator; + + private final RotationRepository rotationRepository; + private final ZmsClientFactory zmsClientFactory; + private final NameService nameService; + private final ConfigServerClient configserverClient; + private final RoutingGenerator routingGenerator; + private final Clock clock; + + private final DeploymentTrigger deploymentTrigger; + + ApplicationController(Controller controller, ControllerDb db, CuratorDb curator, + RotationRepository rotationRepository, + ZmsClientFactory zmsClientFactory, + NameService nameService, ConfigServerClient configserverClient, + RoutingGenerator routingGenerator, Clock clock) { + this.controller = controller; + this.db = db; + this.curator = curator; + this.rotationRepository = rotationRepository; + this.zmsClientFactory = zmsClientFactory; + this.nameService = nameService; + this.configserverClient = configserverClient; + this.routingGenerator = routingGenerator; + this.clock = clock; + + this.deploymentTrigger = new DeploymentTrigger(controller, curator, clock); + + for (Application application : db.listApplications()) { + try (Lock lock = lock(application.id())) { + Optional<Application> optionalApplication = db.getApplication(application.id()); // re-get inside lock + if ( ! optionalApplication.isPresent()) continue; // was removed since listing; ok + store(optionalApplication.get(), lock); // re-write all applications to update storage format + } + } + } + + /** Returns the application with the given id, or null if it is not present */ + public Optional<Application> get(ApplicationId id) { + return db.getApplication(id); + } + + /** + * Returns the application with the given id + * + * @throws IllegalArgumentException if it does not exist + */ + public Application require(ApplicationId id) { + return get(id).orElseThrow(() -> new IllegalArgumentException(id + " not found")); + } + + /** Returns a snapshot of all applications */ + public List<Application> asList() { + return db.listApplications(); + } + + /** Returns all applications of a tenant */ + public List<Application> asList(TenantName tenant) { + return db.listApplications(new TenantId(tenant.value())); + } + + /** + * Set the rotations marked as 'global' either 'in' or 'out of' service. + * + * @return The list of endpoints successfully alertered + * @throws IOException if rotation status cannot be updated + */ + public List<String> setGlobalRotationStatus(DeploymentId deploymentId, EndpointStatus status) throws IOException { + List<String> rotations = new ArrayList<>(); + for (RoutingEndpoint endpoint : routingGenerator.endpoints(deploymentId)) { + if (endpoint.isGlobal()) { + configserverClient.setGlobalRotationStatus(deploymentId, endpoint.getEndpoint(), status); + rotations.add(endpoint.getEndpoint()); + } + } + return rotations; + } + + /** + * Get the endpoint status for rotations marked as 'global' + * + * @return The list of endpoints successfully alertered + * @throws IOException if global rotation status cannot be determined + */ + public Map<String, EndpointStatus> getGlobalRotationStatus(DeploymentId deploymentId) throws IOException { + Map<String, EndpointStatus> result = new HashMap<>(); + for (RoutingEndpoint endpoint : routingGenerator.endpoints(deploymentId)) { + if (endpoint.isGlobal()) { + EndpointStatus status = configserverClient.getGlobalRotationStatus(deploymentId, endpoint.getEndpoint()); + result.put(endpoint.getEndpoint(), status); + } + } + return result; + } + + /** + * Creates a new application for an existing tenant. + * + * @throws IllegalArgumentException if the application already exists + */ + public Application createApplication(ApplicationId id, Optional<NToken> token) { + if ( ! (id.instance().value().equals("default") || id.instance().value().startsWith("default-pr"))) // TODO: Support instances properly + throw new UnsupportedOperationException("Only the instance names 'default' and names starting with 'default-pr' are supported at the moment"); + try (Lock lock = lock(id)) { + if (get(id).isPresent()) + throw new IllegalArgumentException("An application with id '" + id + "' already exists"); + + com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); + + Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(id.tenant().value())); + if ( ! tenant.isPresent()) + throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist"); + if (get(id).isPresent()) + throw new IllegalArgumentException("Could not create '" + id + "': Application already exists"); + if (get(dashToUnderscore(id)).isPresent()) // VESPA-1945 + throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists"); + if (tenant.get().isAthensTenant() && ! token.isPresent()) + throw new IllegalArgumentException("Could not create '" + id + "': No NToken provided"); + if (tenant.get().isAthensTenant()) { + ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()); + try { + zmsClient.deleteApplication(tenant.get().getAthensDomain().get(), + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); + } + catch (ZmsException ignored) { + } + zmsClient.addApplication(tenant.get().getAthensDomain().get(), + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); + } + Application application = new Application(id); + store(application, lock); + log.info("Created " + application); + return application; + } + } + + /** Deploys an application. If the application does not exist it is created. */ + // TODO: Get rid of the options arg + public ActivateResult deployApplication(ApplicationId applicationId, com.yahoo.config.provision.Zone zone, + ApplicationPackage applicationPackage, DeployOptions options) { + try (Lock lock = lock(applicationId)) { + // Determine what we are doing + Application application = get(applicationId).orElse(new Application(applicationId)); + DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); + Version version = decideVersion(application, zone, options); + ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); + + // Ensure that the deploying change is tested + // FIXME: For now only for non-self-triggering applications - VESPA-8418 + if (!application.deploymentJobs().isSelfTriggering() && !zone.environment().isManuallyDeployed() && !application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) { + throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + + " as pending " + application.deploying().get() + + " is untested"); + } + + // Don't update/store applicationpackage information when deploying previous application package (initial staging step) + if(! options.deployCurrentVersion) { + // Add missing information to application + application = application.with(applicationPackage.deploymentSpec()); + application = application.with(applicationPackage.validationOverrides()); + if (options.screwdriverBuildJob.isPresent() && options.screwdriverBuildJob.get().screwdriverId != null) + application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); + if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) + application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); + if (!triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { + // Triggering information is used to store which changes were made or attempted + // - For self-triggered applications we don't have any trigger information, so we add it here. + // - For all applications, we don't have complete control over which revision is actually built, + // so we update it here with what we actually triggered if necessary + application = application.with(application.deploymentJobs().withTriggering(jobType, version, Optional.of(revision), clock.instant())); + } + + store(application, lock); // store missing information even if we fail deployment below + + // Delete zones not listed in DeploymentSpec, if allowed + // We do this at deployment time to be able to return a validation failure message when necessary + application = deleteRemovedDeployments(application); + + // Clean up deployment jobs that are no longer referenced by deployment spec + application = deleteUnreferencedDeploymentJobs(application); + } + + // Carry out deployment + DeploymentId deploymentId = new DeploymentId(applicationId, zone); + ApplicationRotation rotationInDns = registerRotationInDns(deploymentId, getOrAssignRotation(deploymentId, + applicationPackage)); + options = withVersion(version, options); + ConfigServerClient.PreparedApplication preparedApplication = + configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), applicationPackage.zippedContent()); + preparedApplication.activate(); + application = application.with(new Deployment(zone, revision, version, clock.instant())); + store(application, lock); + + return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.messages(), preparedApplication.prepareResponse()); + } + } + + private Version decideVersion(Application application, Zone zone, DeployOptions options) { + if (options.deployCurrentVersion) + return application.currentVersion(controller, zone); + + if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide + return options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); + + if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) + throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + + " as a deployment is not currently expected"); + + return application.currentDeployVersion(controller, zone); + } + + private Application deleteRemovedDeployments(Application application) { + List<Deployment> deploymentsToRemove = application.deployments().values().stream() + .filter(deployment -> deployment.zone().environment() == Environment.prod) + .filter(deployment -> ! application.deploymentSpec().includes(deployment.zone().environment(), + Optional.of(deployment.zone().region()))) + .collect(Collectors.toList()); + + if (deploymentsToRemove.isEmpty()) return application; + + if ( ! application.validationOverrides().allows(ValidationId.deploymentRemoval, clock.instant())) + throw new IllegalArgumentException(ValidationId.deploymentRemoval.value() + ": " + application + + " is deployed in " + + deploymentsToRemove.stream() + .map(deployment -> deployment.zone().region().value()) + .collect(Collectors.joining(", ")) + + ", but does not include " + + (deploymentsToRemove.size() > 1 ? "these zones" : "this zone") + + " in deployment.xml"); + + Application applicationWithRemoval = application; + for (Deployment deployment : deploymentsToRemove) + applicationWithRemoval = deactivate(applicationWithRemoval, deployment, false); + return applicationWithRemoval; + } + + private Application deleteUnreferencedDeploymentJobs(Application application) { + for (DeploymentJobs.JobType job : application.deploymentJobs().jobStatus().keySet()) { + if (!job.isProduction()) { + continue; + } + Optional<Zone> zone = job.zone(controller.system()); + if (!zone.isPresent()) { + continue; + } + if (!application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region))) { + application = application.withoutDeploymentJob(job); + } + } + return application; + } + + private boolean triggeredWith(ApplicationRevision revision, Application application, DeploymentJobs.JobType jobType) { + if (jobType == null) return false; + JobStatus status = application.deploymentJobs().jobStatus().get(jobType); + if (status == null) return false; + if ( ! status.lastTriggered().isPresent()) return false; + JobStatus.JobRun triggered = status.lastTriggered().get(); + if ( ! triggered.revision().isPresent()) return false; + return triggered.revision().get().equals(revision); + } + + private DeployOptions withVersion(Version version, DeployOptions options) { + return new DeployOptions(options.screwdriverBuildJob, + Optional.of(version), + options.ignoreValidationErrors, + options.deployCurrentVersion); + } + + private ApplicationRevision toApplicationPackageRevision(ApplicationPackage applicationPackage, + Optional<ScrewdriverBuildJob> screwDriverBuildJob) { + if ( ! screwDriverBuildJob.isPresent()) + return ApplicationRevision.from(applicationPackage.hash()); + + GitRevision gitRevision = screwDriverBuildJob.get().gitRevision; + if (gitRevision.repository == null || gitRevision.branch == null || gitRevision.commit == null) + return ApplicationRevision.from(applicationPackage.hash()); + + return ApplicationRevision.from(applicationPackage.hash(), new SourceRevision(gitRevision.repository.id(), + gitRevision.branch.id(), + gitRevision.commit.id())); + } + + private ApplicationRotation registerRotationInDns(DeploymentId deploymentId, ApplicationRotation applicationRotation) { + ApplicationAlias alias = new ApplicationAlias(deploymentId.applicationId()); + if (applicationRotation.rotations().isEmpty()) return applicationRotation; + + Rotation rotation = applicationRotation.rotations().iterator().next(); // at this time there should be only one rotation assigned + String endpointName = alias.toString(); + try { + Optional<Record> record = nameService.findRecord(Record.Type.CNAME, rotation.rotationName); + if (!record.isPresent()) { + RecordId recordId = nameService.createCname(endpointName, rotation.rotationName); + log.info("Registered mapping with record ID " + recordId.id() + ": " + + endpointName + " -> " + rotation.rotationName); + } + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to register CNAME", e); + } + return new ApplicationRotation(Collections.singleton(endpointName), Collections.singleton(rotation)); + } + + private ApplicationRotation getOrAssignRotation(DeploymentId deploymentId, ApplicationPackage applicationPackage) { + if (deploymentId.zone().environment().equals(Environment.prod)) { + return new ApplicationRotation(Collections.emptySet(), + rotationRepository.getOrAssignRotation(deploymentId.applicationId(), + applicationPackage.deploymentSpec())); + } else { + return new ApplicationRotation(Collections.emptySet(), + Collections.emptySet()); + } + } + + /** Returns the endpoints of the deployment, or empty if obtaining them failed */ + public Optional<InstanceEndpoints> getDeploymentEndpoints(DeploymentId deploymentId) { + try { + List<RoutingEndpoint> endpoints = routingGenerator.endpoints(deploymentId); + List<URI> endPointUrls = new ArrayList<>(); + for (RoutingEndpoint endpoint : endpoints) { + try { + endPointUrls.add(new URI(endpoint.getEndpoint())); + } catch (URISyntaxException e) { + throw new RuntimeException("Routing generator returned illegal url's", e); + } + } + return Optional.of(new InstanceEndpoints(endPointUrls)); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e); + return Optional.empty(); + } + } + + /** + * Deletes the application with this id + * + * @return the deleted application, or null if it did not exist + * @throws IllegalArgumentException if the application has deployments or the caller is not authorized + */ + public Application deleteApplication(ApplicationId id, Optional<NToken> token) { + try (Lock lock = lock(id)) { + Optional<Application> application = get(id); + if ( ! application.isPresent()) return null; + if ( ! application.get().deployments().isEmpty()) + throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments"); + + Tenant tenant = controller.tenants().tenant(new TenantId(id.tenant().value())).get(); + if (tenant.isAthensTenant() && ! token.isPresent()) + throw new IllegalArgumentException("Could not delete '" + application + "': No NToken provided"); + + // NB: Next 2 lines should have been one transaction + if (tenant.isAthensTenant()) + zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()) + .deleteApplication(tenant.getAthensDomain().get(), new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); + db.deleteApplication(id); + + log.info("Deleted " + application.get()); + return application.get(); + } + } + + public void setJiraIssueId(ApplicationId id, Optional<String> jiraIssueId) { + try (Lock lock = lock(id)) { + get(id).ifPresent(application -> store(application.withJiraIssueId(jiraIssueId), lock)); + } + } + + /** + * Replace any previous version of this application by this instance + * + * @param application the application version to store + * @param lock the lock held on this application since before modification started + */ + @SuppressWarnings("unused") // lock is part of the signature to remind people to acquire it, not needed internally + public void store(Application application, Lock lock) { + db.store(application); + } + + public void notifyJobCompletion(JobReport report) { + if ( ! get(report.applicationId()).isPresent()) { + log.log(Level.WARNING, "Ignoring completion of job of project '" + report.projectId() + + "': Unknown application '" + report.applicationId() + "'"); + return; + } + deploymentTrigger.triggerFromCompletion(report); + } + + // TODO: Collapse this method and the next + public void restart(DeploymentId deploymentId) { + try { + configserverClient.restart(deploymentId, Optional.empty()); + } + catch (NoInstanceException e) { + throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment"); + } + } + public void restartHost(DeploymentId deploymentId, Hostname hostname) { + try { + configserverClient.restart(deploymentId, Optional.of(hostname)); + } + catch (NoInstanceException e) { + throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment"); + } + } + + public Application deactivate(Application application, Deployment deployment, boolean requireThatDeploymentHasExpired) { + try (Lock lock = lock(application.id())) { + // TODO: ignore no application errors for config server client, + // only return such errors from sherpa client. + if (requireThatDeploymentHasExpired && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment, + clock.instant())) + return application; + + try { + configserverClient.deactivate(new DeploymentId(application.id(), deployment.zone())); + } + catch (NoInstanceException e) { + // ok; already gone + } + application = application.withoutDeploymentIn(deployment.zone()); + store(application, lock); + return application; + } + } + + public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; } + + private ApplicationId dashToUnderscore(ApplicationId id) { + return ApplicationId.from(id.tenant().value(), + id.application().value().replaceAll("-", "_"), + id.instance().value()); + } + + public ConfigServerClient configserverClient() { return configserverClient; } + + /** + * Returns a lock which provides exclusive rights to changing this application. + * Any operation which stores an application need to first acquire this lock, then read, modify + * and store the application, and finally release (close) the lock. + */ + public Lock lock(ApplicationId application) { + return curator.lock(application, Duration.ofMinutes(10)); + } + + private static final class ApplicationRotation { + + private final ImmutableSet<String> cnames; + private final ImmutableSet<Rotation> rotations; + + public ApplicationRotation(Set<String> cnames, Set<Rotation> rotations) { + this.cnames = ImmutableSet.copyOf(cnames); + this.rotations = ImmutableSet.copyOf(rotations); + } + + public Set<String> cnames() { return cnames; } + public Set<Rotation> rotations() { return rotations; } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java new file mode 100644 index 00000000000..dcb54f13e4b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -0,0 +1,273 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; +import com.yahoo.vespa.hosted.controller.api.integration.cost.Cost; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; +import com.yahoo.vespa.hosted.controller.api.integration.jira.Jira; +import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import com.yahoo.vespa.hosted.rotation.RotationRepository; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; + +import java.net.URI; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +/** + * API to the controller. This contains (currently: should contain) the object model of everything the + * controller cares about, mainly tenants and applications. + * + * As the controller runtime and Controller object are singletons, this instance can read from the object model + * in memory. However, all changes to the object model must be persisted in the controller db. + * + * All the individual model objects reachable from the Controller are immutable. + * + * Access to the controller is multithread safe, provided the locking methods are + * used when accessing, modifying and storing objects provided by the controller. + * + * @author bratseth + */ +public class Controller extends AbstractComponent { + + private static final Logger log = Logger.getLogger(Controller.class.getName()); + + private final CuratorDb curator; + private final ApplicationController applicationController; + private final TenantController tenantController; + + /** + * Status of Vespa versions across the system. + * This is expensive to maintain so that is done periodically by a maintenance job + */ + private final AtomicReference<VersionStatus> versionStatus; + + private final Clock clock; + + private final RotationRepository rotationRepository; + private final GitHub gitHub; + private final EntityService entityService; + private final GlobalRoutingService globalRoutingService; + private final ZoneRegistry zoneRegistry; + private final Cost cost; + private final ConfigServerClient configServerClient; + private final MetricsService metricsService; + private final Chef chefClient; + private final Athens athens; + + /** + * Creates a controller + * + * @param db the db storing persistent state + * @param curator the curator instance storing working state shared between controller instances + */ + @Inject + public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository, + GitHub gitHub, Jira jiraClient, EntityService entityService, + GlobalRoutingService globalRoutingService, + ZoneRegistry zoneRegistry, Cost cost, ConfigServerClient configServerClient, + MetricsService metricsService, NameService nameService, + RoutingGenerator routingGenerator, Chef chefClient, Athens athens) { + this(db, curator, rotationRepository, + gitHub, jiraClient, entityService, globalRoutingService, zoneRegistry, + cost, configServerClient, metricsService, nameService, routingGenerator, chefClient, + Clock.systemUTC(), athens); + } + + public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository, + GitHub gitHub, Jira jiraClient, EntityService entityService, + GlobalRoutingService globalRoutingService, + ZoneRegistry zoneRegistry, Cost cost, ConfigServerClient configServerClient, + MetricsService metricsService, NameService nameService, + RoutingGenerator routingGenerator, Chef chefClient, Clock clock, Athens athens) { + Objects.requireNonNull(db, "Controller db cannot be null"); + Objects.requireNonNull(curator, "Curator cannot be null"); + Objects.requireNonNull(rotationRepository, "Rotation repository cannot be null"); + Objects.requireNonNull(gitHub, "GitHubClient cannot be null"); + Objects.requireNonNull(jiraClient, "JiraClient cannot be null"); + Objects.requireNonNull(entityService, "EntityService cannot be null"); + Objects.requireNonNull(globalRoutingService, "GlobalRoutingService cannot be null"); + Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); + Objects.requireNonNull(cost, "Cost cannot be null"); + Objects.requireNonNull(configServerClient, "ConfigServerClient cannot be null"); + Objects.requireNonNull(metricsService, "MetricsService cannot be null"); + Objects.requireNonNull(nameService, "NameService cannot be null"); + Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"); + Objects.requireNonNull(chefClient, "ChefClient cannot be null"); + Objects.requireNonNull(clock, "Clock cannot be null"); + Objects.requireNonNull(athens, "Athens cannot be null"); + + this.rotationRepository = rotationRepository; + this.curator = curator; + this.gitHub = gitHub; + this.entityService = entityService; + this.globalRoutingService = globalRoutingService; + this.zoneRegistry = zoneRegistry; + this.cost = cost; + this.configServerClient = configServerClient; + this.metricsService = metricsService; + this.chefClient = chefClient; + this.clock = clock; + this.athens = athens; + + applicationController = new ApplicationController(this, db, curator, rotationRepository, athens.zmsClientFactory(), + nameService, configServerClient, routingGenerator, clock); + tenantController = new TenantController(this, db, curator, entityService); + versionStatus = new AtomicReference<>(VersionStatus.empty()); + } + + /** Returns the instance controlling tenants */ + public TenantController tenants() { return tenantController; } + + /** Returns the instance controlling applications */ + public ApplicationController applications() { return applicationController; } + + public List<AthensDomain> getDomainList(String prefix) { + return athens.unauthorizedZmsClient().getDomainList(prefix); + } + + public Athens athens() { + return athens; + } + + /** + * Fetch list of all active OpsDB properties. + * + * @return Hashed map with the property ID as key and property name as value + */ + public Map<PropertyId, Property> fetchPropertyList() { + return entityService.listProperties(); + } + + public Clock clock() { return clock; } + + public ApplicationCost getApplicationCost(com.yahoo.config.provision.ApplicationId application, + com.yahoo.config.provision.Zone zone) + throws NotFoundCheckedException { + return cost.getApplicationCost(zone.environment(), zone.region(), application); + } + + public URI getElkUri(Environment environment, RegionName region, DeploymentId deploymentId) { + return elkUrl(zoneRegistry.getLogServerUri(environment, region), deploymentId); + } + + public List<URI> getConfigServerUris(Environment environment, RegionName region) { + return zoneRegistry.getConfigServerUris(environment, region); + } + + public ZoneRegistry zoneRegistry() { return zoneRegistry; } + + private URI elkUrl(Optional<URI> kibanaHost, DeploymentId deploymentId) { + String kibanaQuery = "/#/discover?_g=()&_a=(columns:!(_source)," + + "index:'logstash-*',interval:auto," + + "query:(query_string:(analyze_wildcard:!t,query:'" + + "HV-tenant:%22" + deploymentId.applicationId().tenant().value() + "%22%20" + + "AND%20HV-application:%22" + deploymentId.applicationId().application().value() + "%22%20" + + "AND%20HV-region:%22" + deploymentId.zone().region().value() + "%22%20" + + "AND%20HV-instance:%22" + deploymentId.applicationId().instance().value() + "%22%20" + + "AND%20HV-environment:%22" + deploymentId.zone().environment().value() + "%22'))," + + "sort:!('@timestamp',desc))"; + + URI kibanaPath = URI.create(kibanaQuery); + if (kibanaHost.isPresent()) { + return kibanaHost.get().resolve(kibanaPath); + } else { + return null; + } + } + + public Set<URI> getRotationUris(ApplicationId id) { + return rotationRepository.getRotationUris(id); + } + + public Map<String, RotationStatus> getHealthStatus(String hostname) { + return globalRoutingService.getHealthStatus(hostname); + } + + // TODO: Model the response properly + public JsonNode waitForConfigConvergence(DeploymentId deploymentId, long timeout) { + return configServerClient.waitForConfigConverge(deploymentId, timeout); + } + + public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { + return configServerClient.getApplicationView(tenantName, applicationName, instanceName, environment, region); + } + + // TODO: Model the response properly + public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath) { + return configServerClient.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); + } + + // TODO: Model the response properly + // TODO: What is this + public JsonNode grabLog(DeploymentId deploymentId) { + return configServerClient.grabLog(deploymentId); + } + + public GitHub gitHub() { return gitHub; } + + /** Replace the current version status by a new one */ + public void updateVersionStatus(VersionStatus newStatus) { + VersionStatus currentStatus = versionStatus(); + if (newStatus.systemVersion().isPresent() && + ! newStatus.systemVersion().equals(currentStatus.systemVersion())) { + log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) + + " to " + printableVersion(newStatus.systemVersion())); + curator.writeSystemVersion(newStatus.systemVersion().get().versionNumber()); + } + + this.versionStatus.set(newStatus); + } + + /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */ + public VersionStatus versionStatus() { return versionStatus.get(); } + + /** Returns the current system version: The controller should drive towards running all applications on this version */ + public Version systemVersion() { return curator.readSystemVersion(); } + + public MetricsService metricsService() { return metricsService; } + + public SystemName system() { + return zoneRegistry.system(); + } + + public Chef chefClient() { + return chefClient; + } + + private String printableVersion(Optional<VespaVersion> vespaVersion) { + return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("Unknown"); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java new file mode 100644 index 00000000000..6a47957f27f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; + +/** + * An exception which indicates that a requested resource does not exist. + * + * @author Tony Vaagenes + */ +public class NotExistsException extends IllegalArgumentException { + + public NotExistsException(String message) { + super(message); + } + + /** + * Example message: Tenant 'myId' does not exist. + * + * @param capitalizedType e.g. Tenant, Application + * @param id The id of the entity that didn't exist. + * + */ + public NotExistsException(String capitalizedType, String id) { + super(String.format("%s '%s' does not exist", capitalizedType, id)); + } + + public NotExistsException(Identifier id) { + this(id.capitalizedType(), id.id()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java new file mode 100644 index 00000000000..fafd0b04dd2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -0,0 +1,238 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; + +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A singleton owned by the Controller which contains the methods and state for controlling applications. + * + * @author bratseth + */ +public class TenantController { + + private static final Logger log = Logger.getLogger(TenantController.class.getName()); + + /** The controller owning this */ + private final Controller controller; + + /** For permanent storage */ + private final ControllerDb db; + + /** For working memory storage and sharing between controllers */ + private final CuratorDb curator; + + private final ZmsClientFactory zmsClientFactory; + private final EntityService entityService; + + public TenantController(Controller controller, ControllerDb db, CuratorDb curator, EntityService entityService) { + this.controller = controller; + this.db = db; + this.curator = curator; + this.zmsClientFactory = controller.athens().zmsClientFactory(); + this.entityService = entityService; + } + + public List<Tenant> asList() { + return db.listTenants(); + } + + public List<Tenant> asList(UserId user) { + Set<UserGroup> userGroups = entityService.getUserGroups(user); + Set<AthensDomain> userDomains = new HashSet<>(zmsClientFactory.createClientWithServicePrincipal() + .getTenantDomainsForUser(controller.athens().principalFrom(user))); + + Predicate<Tenant> hasUsersGroup = (tenant) -> tenant.getUserGroup().isPresent() && userGroups.contains(tenant.getUserGroup().get()); + Predicate<Tenant> hasUsersDomain = (tenant) -> tenant.getAthensDomain().isPresent() && userDomains.contains(tenant.getAthensDomain().get()); + Predicate<Tenant> isUserTenant = (tenant) -> tenant.getId().equals(user.toTenantId()); + + return asList().stream() + .filter(t -> hasUsersGroup.test(t) || hasUsersDomain.test(t) || isUserTenant.test(t)) + .collect(Collectors.toList()); + } + + public Tenant createUserTenant(String userName) { + TenantId userTenantId = new UserId(userName).toTenantId(); + try (Lock lock = lock(userTenantId)) { + Tenant tenant = Tenant.createUserTenant(userTenantId); + internalCreateTenant(tenant, Optional.empty()); + return tenant; + } + } + + /** Creates an Athens or OpsDb tenant. */ + // TODO: Rename to createAthensTenant and move creation here when opsDbTenant creation is removed */ + public void addTenant(Tenant tenant, Optional<NToken> token) { + try (Lock lock = lock(tenant.getId())) { + internalCreateTenant(tenant, token); + } + } + + private void internalCreateTenant(Tenant tenant, Optional<NToken> token) { + TenantId.validate(tenant.getId().id()); + if (tenant(tenant.getId()).isPresent()) + throw new IllegalArgumentException("Tenant '" + tenant.getId() + "' already exists"); + if (tenant(dashToUnderscore(tenant.getId())).isPresent()) + throw new IllegalArgumentException("Could not create " + tenant + ": Tenant " + dashToUnderscore(tenant.getId()) + " already exists"); + if (tenant.isAthensTenant() && ! token.isPresent()) + throw new IllegalArgumentException("Could not create " + tenant + ": No NToken provided"); + + if (tenant.isAthensTenant()) { + AthensDomain domain = tenant.getAthensDomain().get(); + Optional<Tenant> existingTenantWithDomain = tenantHaving(domain); + if (existingTenantWithDomain.isPresent()) + throw new IllegalArgumentException("Could not create " + tenant + ": The Athens domain '" + domain + + "' is already connected to " + existingTenantWithDomain.get()); + ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()); + try { zmsClient.deleteTenant(domain); } catch (ZmsException ignored) { } + zmsClient.createTenant(domain); + } + db.createTenant(tenant); + log.info("Created " + tenant); + } + + /** Returns the tenant having the given Athens domain, or empty if none */ + private Optional<Tenant> tenantHaving(AthensDomain domain) { + return asList().stream().filter(Tenant::isAthensTenant) + .filter(t -> t.getAthensDomain().get().equals(domain)) + .findAny(); + } + + public Optional<Tenant> tenant(TenantId id) { + try { + return db.getTenant(id); + } catch (PersistenceException e) { + throw new RuntimeException(e); + } + } + + public void updateTenant(Tenant updatedTenant, Optional<NToken> token) { + try (Lock lock = lock(updatedTenant.getId())) { + if ( ! tenant(updatedTenant.getId()).isPresent()) + throw new IllegalArgumentException("Could not update " + updatedTenant + ": Tenant does not exist"); + if (updatedTenant.isAthensTenant() && ! token.isPresent()) + throw new IllegalArgumentException("Could not update " + updatedTenant + ": No NToken provided"); + + updateAthensDomain(updatedTenant, token); + db.updateTenant(updatedTenant); + log.info("Updated " + updatedTenant); + } catch (PersistenceException e) { + throw new RuntimeException(e); + } + } + + private void updateAthensDomain(Tenant updatedTenant, Optional<NToken> token) { + Tenant existingTenant = tenant(updatedTenant.getId()).get(); + if ( ! existingTenant.isAthensTenant()) return; + + AthensDomain existingDomain = existingTenant.getAthensDomain().get(); + AthensDomain newDomain = updatedTenant.getAthensDomain().get(); + if (existingDomain.equals(newDomain)) return; + Optional<Tenant> existingTenantWithNewDomain = tenantHaving(newDomain); + if (existingTenantWithNewDomain.isPresent()) + throw new IllegalArgumentException("Could not set domain of " + updatedTenant + " to '" + newDomain + + "':" + existingTenantWithNewDomain.get() + " already has this domain"); + + ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()); + zmsClient.createTenant(newDomain); + List<Application> applications = controller.applications().asList(TenantName.from(existingTenant.getId().id())); + applications.forEach(a -> zmsClient.addApplication(newDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()))); + applications.forEach(a -> zmsClient.deleteApplication(existingDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()))); + zmsClient.deleteTenant(existingDomain); + log.info("Updated Athens domain for " + updatedTenant + " from " + existingDomain + " to " + newDomain); + } + + public void deleteTenant(TenantId id, Optional<NToken> token) { + try (Lock lock = lock(id)) { + if ( ! tenant(id).isPresent()) + throw new NotExistsException(id); // TODO: Change exception and message + if ( ! controller.applications().asList(TenantName.from(id.id())).isEmpty()) + throw new IllegalArgumentException("Could not delete tenant '" + id + "': This tenant has active applications"); + + Tenant tenant = tenant(id).get(); + if (tenant.isAthensTenant() && ! token.isPresent()) + throw new IllegalArgumentException("Could not delete tenant '" + id + "': No NToken provided"); + + try { + db.deleteTenant(id); + } catch (PersistenceException e) { // TODO: Don't allow these to leak out + throw new RuntimeException(e); + } + if (tenant.isAthensTenant()) + zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()).deleteTenant(tenant.getAthensDomain().get()); + log.info("Deleted " + tenant); + } + } + + public Tenant migrateTenantToAthens(TenantId tenantId, + AthensDomain tenantDomain, + PropertyId propertyId, + Property property, + NToken nToken) { + try (Lock lock = lock(tenantId)) { + Tenant existing = tenant(tenantId).orElseThrow(() -> new NotExistsException(tenantId)); + if (existing.isAthensTenant()) return existing; // nothing to do + if (tenantHaving(tenantDomain).isPresent()) + throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + + "This domain is already used by " + tenantHaving(tenantDomain).get()); + if ( ! existing.isOpsDbTenant()) + throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + + "Tenant is not currently an OpsDb tenant"); + + ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(nToken); + zmsClient.createTenant(tenantDomain); + List<Application> applications = controller.applications().asList(TenantName.from(existing.getId().id())); + applications.forEach(a -> { + ApplicationId applicationId = new ApplicationId(a.id().application().value()); + zmsClient.addApplication(tenantDomain, applicationId); + }); + db.deleteTenant(tenantId); + Tenant tenant = Tenant.createAthensTenant(tenantId, tenantDomain, property, Optional.of(propertyId)); + db.createTenant(tenant); + log.info("Migrated " + existing + " to Athens using " + tenantDomain); + return tenant; + } + catch (PersistenceException e) { + throw new RuntimeException("Failed migrating " + tenantId + " to Athens", e); + } + } + + private TenantId dashToUnderscore(TenantId id) { + return new TenantId(id.id().replaceAll("-", "_")); + } + + /** + * Returns a lock which provides exclusive rights to changing this tenant. + * Any operation which stores a tenant need to first acquire this lock, then read, modify + * and store the tenant, and finally release (close) the lock. + */ + private Lock lock(TenantId tenant) { + return curator.lock(tenant, Duration.ofMinutes(10)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java new file mode 100644 index 00000000000..1fb6a4a8582 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java @@ -0,0 +1,37 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api; + +import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; + +import java.util.List; + +/** + * @author Oyvind Gronnesby + */ +public class ActivateResult { + + private final RevisionId revisionId; + private final List<Log> messages; + private final PrepareResponse prepareResponse; + + public ActivateResult(RevisionId revisionId, List<Log> messages, PrepareResponse prepareResponse) { + this.revisionId = revisionId; + this.messages = messages; + this.prepareResponse = prepareResponse; + } + + public RevisionId getRevisionId() { + return revisionId; + } + + public List<Log> getMessages() { + return messages; + } + + public PrepareResponse getPrepareResponse() { + return prepareResponse; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java new file mode 100644 index 00000000000..a9e144a3227 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java @@ -0,0 +1,57 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api; + +import com.yahoo.config.provision.ApplicationId; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * A DNS alias for an application endpoint. + * + * @author smorgrav + */ +public class ApplicationAlias { + + private static final String dnsSuffix = "global.vespa.yahooapis.com"; + + private final ApplicationId applicationId; + + public ApplicationAlias(ApplicationId applicationId) { + this.applicationId = applicationId; + } + + @Override + public String toString() { + return String.format("%s.%s.%s", + toDns(applicationId.application().value()), + toDns(applicationId.tenant().value()), + dnsSuffix); + } + + private String toDns(String id) { + return id.replace('_', '-'); + } + + public URI toHttpUri() { + try { + return new URI("http://" + this + ":4080/"); + } catch(URISyntaxException use) { + throw new RuntimeException("Illegal URI syntax"); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ApplicationAlias that = (ApplicationAlias) o; + + return applicationId.equals(that.applicationId); + } + + @Override + public int hashCode() { return applicationId.hashCode(); } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java new file mode 100644 index 00000000000..b9ed439eb8b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api; + +import java.net.URI; +import java.util.List; + +/** + * @author Tony Vaagenes + */ +public class InstanceEndpoints { + + private final List<URI> containerEndpoints; + + public InstanceEndpoints(List<URI> containerEndpoints) { + this.containerEndpoints = containerEndpoints; + } + + public List<URI> getContainerEndpoints() { + return containerEndpoints; + } +} + + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java new file mode 100644 index 00000000000..325c40c24c8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java @@ -0,0 +1,147 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api; + +import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; + +import java.util.Optional; + +/** + * @author smorgrav + */ +// TODO: Move this and everything it owns to com.yahoo.hosted.controller.Tenant and com.yahoo.hosted.controller.tenant.* +public class Tenant { + + private final TenantId id; + private final Optional<UserGroup> userGroup; + private final Optional<Property> property; + private final Optional<AthensDomain> athensDomain; + private final Optional<PropertyId> propertyId; + + // TODO: Use factory methods. They're down at the bottom! + public Tenant(TenantId id, Optional<UserGroup> userGroup, Optional<Property> property, Optional<AthensDomain> athensDomain) { + this(id, userGroup, property, athensDomain, Optional.empty()); + } + + public Tenant(TenantId id, Optional<UserGroup> userGroup, Optional<Property> property, Optional<AthensDomain> athensDomain, Optional<PropertyId> propertyId) { + if (id.isUser()) { + require(!userGroup.isPresent(), "User tenant '%s' cannot have a user group.", id); + require(!property.isPresent(), "User tenant '%s' cannot have a property.", id); + require(!propertyId.isPresent(), "User tenant '%s' cannot have a property ID.", id); + require(!athensDomain.isPresent(), "User tenant '%s' cannot have an athens domain.", id); + } else if (athensDomain.isPresent()) { + require(property.isPresent(), "Athens tenant '%s' must have a property.", id); + require(!userGroup.isPresent(), "Athens tenant '%s' cannot have a user group.", id); + require(athensDomain.isPresent(), "Athens tenant '%s' must have an athens domain.", id); + } else { + require(property.isPresent(), "OpsDB tenant '%s' must have a property.", id); + require(userGroup.isPresent(), "OpsDb tenant '%s' must have a user group.", id); + require(!athensDomain.isPresent(), "OpsDb tenant '%s' cannot have an athens domain.", id); + } + this.id = id; + this.userGroup = userGroup; + this.property = property; + this.athensDomain = athensDomain; + this.propertyId = propertyId; // TODO: Check validity after TODO@14. OpsDb tenants have this set in Sherpa, while athens tenants do not. + } + + public boolean isAthensTenant() { return athensDomain.isPresent(); } + public boolean isOpsDbTenant() { return userGroup.isPresent();} + + public TenantType tenantType() { + if (athensDomain.isPresent()) { + return TenantType.ATHENS; + } else if (id.isUser()) { + return TenantType.USER; + } else { + return TenantType.OPSDB; + } + } + + public TenantId getId() { + return id; + } + + public Optional<UserGroup> getUserGroup() { + return userGroup; + } + + /** OpsDB property name of the tenant, or Optional.empty() if none is stored. */ + public Optional<Property> getProperty() { + return property; + } + + /** OpsDB property ID of the tenant. Not (yet) required, so returns Optional.empty() if none is stored. */ + public Optional<PropertyId> getPropertyId() { + return propertyId; + } + + public Optional<AthensDomain> getAthensDomain() { + return athensDomain; + } + + private void require(boolean statement, String message, TenantId id) { + if (!statement) throw new IllegalArgumentException(String.format(message, id)); + } + + public static Tenant createAthensTenant(TenantId id, AthensDomain athensDomain, Property property, Optional<PropertyId> propertyId) { + if (id.isUser()) { + throw new IllegalArgumentException("Invalid id for non-user tenant: " + id); + } + return new Tenant(id, Optional.empty(), Optional.ofNullable(property), + Optional.ofNullable(athensDomain), propertyId); + } + + public static Tenant createOpsDbTenant(TenantId id, UserGroup userGroup, Property property, Optional<PropertyId> propertyId) { + if (id.isUser()) { + throw new IllegalArgumentException("Invalid id for non-user tenant: " + id); + } + return new Tenant(id, Optional.ofNullable(userGroup), Optional.ofNullable(property), Optional.empty(), propertyId); + } + + public static Tenant createOpsDbTenant(TenantId id, UserGroup userGroup, Property property) { + return createOpsDbTenant(id, userGroup, property, Optional.empty()); + } + + public static Tenant createUserTenant(TenantId id) { + if (!id.isUser()) { + throw new IllegalArgumentException("Invalid id for user tenant: " + id); + } + return new Tenant(id, Optional.empty(), Optional.empty(), Optional.empty()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Tenant tenant = (Tenant) o; + + if (!id.equals(tenant.id)) return false; + if (!userGroup.equals(tenant.userGroup)) return false; + if (!property.equals(tenant.property)) return false; + if (!athensDomain.equals(tenant.athensDomain)) return false; + if (!propertyId.equals(tenant.propertyId)) return false; + return true; + } + + @Override + public int hashCode() { + int result = id.hashCode(); + result = 31 * result + userGroup.hashCode(); + result = 31 * result + property.hashCode(); + result = 31 * result + athensDomain.hashCode(); + result = 31 * result + propertyId.hashCode(); + return result; + } + + @Override + public String toString() { + return "tenant '" + id + "'"; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java new file mode 100644 index 00000000000..4b405f55e10 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Tony Vaagenes + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.api; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java new file mode 100644 index 00000000000..3fcd285e0fc --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -0,0 +1,200 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; + +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +/** + * A list of applications which can be filtered in various ways. + * + * @author bratseth + */ +public class ApplicationList { + + private final ImmutableList<Application> list; + + private ApplicationList(List<Application> applications) { + this.list = ImmutableList.copyOf(applications); + } + + // ----------------------------------- Factories + + public static ApplicationList from(List<Application> applications) { + return new ApplicationList(applications); + } + + public static ApplicationList from(List<ApplicationId> ids, ApplicationController applications) { + return listOf(ids.stream().map(applications::require)); + } + + // ----------------------------------- Accessors + + /** Returns the applications in this as an immutable list */ + public List<Application> asList() { return list; } + + public boolean isEmpty() { return list.isEmpty(); } + + public int size() { return list.size(); } + + // ----------------------------------- Filters + + /** Returns the subset of applications which is currently upgrading to the given version */ + public ApplicationList upgradingTo(Version version) { + return listOf(list.stream().filter(application -> isUpgradingTo(version, application))); + } + + /** Returns the subset of applications which is currently upgrading to a version lower than the given version */ + public ApplicationList upgradingToLowerThan(Version version) { + return listOf(list.stream().filter(application -> isUpgradingToLowerThan(version, application))); + } + + /** Returns the subset of applications which is currently not upgrading to the given version */ + public ApplicationList notUpgradingTo(Version version) { + return listOf(list.stream().filter(application -> ! isUpgradingTo(version, application))); + } + + /** Returns the subset of applications which is currently not deploying a new application revision */ + public ApplicationList notDeployingApplication() { + return listOf(list.stream().filter(application -> ! isDeployingApplicationChange(application))); + } + + /** Returns the subset of applications which currently does not have any failing jobs */ + public ApplicationList notFailing() { + return listOf(list.stream().filter(application -> ! application.deploymentJobs().hasFailures())); + } + + /** Returns the subset of applications which currently does not have any failing jobs on the given version */ + public ApplicationList notFailingOn(Version version) { + return listOf(list.stream().filter(application -> ! failingOn(version, application))); + } + + /** Returns the subset of applications which have one or more deployment jobs failing for the current change */ + public ApplicationList hasDeploymentFailures() { + return listOf(list.stream().filter(application -> application.deploying().isPresent() && application.deploymentJobs().failingOn(application.deploying().get()))); + } + + /** Returns the subset of applications which have at least one deployment */ + public ApplicationList hasDeployment() { + return listOf(list.stream().filter(a -> !a.deployments().isEmpty())); + } + + /** Returns the subset of applications that are currently deploying a change */ + public ApplicationList isDeploying() { + return listOf(list.stream().filter(application -> application.deploying().isPresent())); + } + + /** Returns the subset of applications which started failing after the given instant */ + public ApplicationList startedFailingAfter(Instant instant) { + return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant))); + } + + /** Returns the subset of applications which has the given upgrade policy */ + public ApplicationList with(UpgradePolicy policy) { + return listOf(list.stream().filter(a -> a.deploymentSpec().upgradePolicy() == policy)); + } + + /** Returns the subset of applications which does not have the given upgrade policy */ + public ApplicationList without(UpgradePolicy policy) { + return listOf(list.stream().filter(a -> a.deploymentSpec().upgradePolicy() != policy)); + } + + /** Returns the subset of applications which have at least one deployment on a lower version than the given one */ + public ApplicationList onLowerVersionThan(Version version) { + return listOf(list.stream() + .filter(a -> a.deployments().values().stream().anyMatch(d -> d.version().isBefore(version)))); + } + + /** + * Returns the subset of applications which are not pull requests: + * Pull requests changes the application instance name to default-pr[pull-request-number] + */ + public ApplicationList notPullRequest() { + return listOf(list.stream().filter(a -> ! a.id().instance().value().startsWith("default-pr"))); + } + + // ----------------------------------- Sorting + + /** + * Returns this list sorted by increasing deployed version. + * If multiple versions are deployed the oldest is used. + * Applications without any deployments are ordered first. + */ + public ApplicationList byIncreasingDeployedVersion() { + return listOf(list.stream().sorted(Comparator.comparing(application -> application.deployedVersion().orElse(Version.emptyVersion)))); + } + + /** Returns the subset of applications which currently do not have any job in progress for the given change */ + public ApplicationList notRunningJobFor(Change.VersionChange change) { + return listOf(list.stream().filter(a -> !hasRunningJob(a, change))); + } + + /** Returns the subset of applications which currently do not have any job in progress */ + public ApplicationList notRunningJob() { + return listOf(list.stream().filter(a -> !a.deploymentJobs().inProgress())); + } + + /** Returns the subset of applications which has a job that started running before the given instant */ + public ApplicationList jobRunningSince(Instant instant) { + return listOf(list.stream().filter(a -> a.deploymentJobs().runningSince() + .map(at -> at.isBefore(instant)) + .orElse(false))); + } + + /** Returns the subset of applications which deploys to given environment and region */ + public ApplicationList deploysTo(Environment environment, RegionName region) { + return listOf(list.stream().filter(a -> a.deploymentSpec().includes(environment, Optional.of(region)))); + } + + // ----------------------------------- Internal helpers + + private static boolean isUpgradingTo(Version version, Application application) { + if ( ! (application.deploying().isPresent()) ) return false; + if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false; + return ((Change.VersionChange)application.deploying().get()).version().equals(version); + } + + private static boolean isUpgradingToLowerThan(Version version, Application application) { + if ( ! application.deploying().isPresent()) return false; + if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false; + return ((Change.VersionChange)application.deploying().get()).version().isBefore(version); + } + + private static boolean isDeployingApplicationChange(Application application) { + if ( ! application.deploying().isPresent()) return false; + return application.deploying().get() instanceof Change.ApplicationChange; + } + + private static boolean failingOn(Version version, Application application) { + for (JobStatus jobStatus : application.deploymentJobs().jobStatus().values()) + if ( ! jobStatus.isSuccess() && jobStatus.lastCompleted().get().version().equals(version)) return true; + return false; + } + + private static boolean hasRunningJob(Application application, Change.VersionChange change) { + return application.deploymentJobs().jobStatus().values().stream() + .filter(JobStatus::inProgress) + .filter(jobStatus -> jobStatus.lastTriggered().isPresent()) + .map(jobStatus -> jobStatus.lastTriggered().get()) + .anyMatch(jobRun -> jobRun.version().equals(change.version())); + } + + /** Convenience converter from a stream to an ApplicationList */ + private static ApplicationList listOf(Stream<Application> applications) { + ImmutableList.Builder<Application> b = new ImmutableList.Builder<>(); + applications.forEach(b::add); + return new ApplicationList(b.build()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java new file mode 100644 index 00000000000..6df8e901653 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java @@ -0,0 +1,75 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationOverrides; +import org.apache.commons.codec.digest.DigestUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Objects; +import java.util.Optional; + +/** + * A representation of the content of an application package. + * Only the deployment.xml content can be accessed as anything other than compressed data. + * A package is identified by a hash of the content. + * + * This is immutable. + * + * @author bratseth + */ +public class ApplicationPackage { + + private final String contentHash; + private final byte[] zippedContent; + private final DeploymentSpec deploymentSpec; + private final ValidationOverrides validationOverrides; + + /** + * Creates an application package from its zipped content. + * This <b>assigns ownership</b> of the given byte array to this class: + * it must not be further changed by the caller. + */ + public ApplicationPackage(byte[] zippedContent) { + Objects.requireNonNull(zippedContent, "The application package content cannot be null"); + this.contentHash = DigestUtils.shaHex(zippedContent); + this.zippedContent = zippedContent; + this.deploymentSpec = extractFile("deployment.xml", zippedContent).map(DeploymentSpec::fromXml).orElse(DeploymentSpec.empty); + this.validationOverrides = extractFile("validation-overrides.xml", zippedContent).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); + } + + /** Returns a hash of the content of this package */ + public String hash() { return contentHash; } + + /** Returns the content of this package. The content <b>must not</b> be modified. */ + public byte[] zippedContent() { return zippedContent; } + + /** + * Returns the deployment spec from the deployment.xml file of the package content. + * This is the DeploymentSpec.empty instance if this package does not contain a deployment.xml file. + */ + public DeploymentSpec deploymentSpec() { return deploymentSpec; } + + /** + * Returns the validation overrides from the validation-overrides.xml file of the package content. + * This is the ValidationOverrides.empty instance if this package does not contain a validation-overrides.xml file. + */ + public ValidationOverrides validationOverrides() { return validationOverrides; } + + private static Optional<Reader> extractFile(String fileName, byte[] zippedContent) { + try (ByteArrayInputStream stream = new ByteArrayInputStream(zippedContent)) { + ZipStreamReader reader = new ZipStreamReader(stream); + for (ZipStreamReader.ZipEntryWithContent entry : reader.entries()) + if (entry.zipEntry().getName().equals(fileName) || entry.zipEntry().getName().equals("application/" + fileName)) // TODO: Remove application/ directory support + return Optional.of(new InputStreamReader(new ByteArrayInputStream(entry.content()))); + return Optional.empty(); + } + catch (IOException e) { + throw new IllegalArgumentException("Exception reading application package", e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java new file mode 100644 index 00000000000..1b875f28715 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java @@ -0,0 +1,60 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import java.util.Objects; +import java.util.Optional; + +/** + * An identifier of a particular revision (exact content) of an application package, + * optionally with information about the source of the package revision. + * + * @author bratseth + */ +public class ApplicationRevision { + + private final String applicationPackageHash; + + private final Optional<SourceRevision> source; + + private ApplicationRevision(String applicationPackageHash, Optional<SourceRevision> source) { + Objects.requireNonNull(applicationPackageHash, "applicationPackageHash cannot be null"); + this.applicationPackageHash = applicationPackageHash; + this.source = source; + } + + /** Create an application package revision where there is no information about its source */ + public static ApplicationRevision from(String applicationPackageHash) { + return new ApplicationRevision(applicationPackageHash, Optional.empty()); + } + + /** Create an application package revision with a source */ + public static ApplicationRevision from(String applicationPackageHash, SourceRevision source) { + return new ApplicationRevision(applicationPackageHash, Optional.of(source)); + } + + /** Returns a unique, content-based identifier of an application package (a hash of the content) */ + public String id() { return applicationPackageHash; } + + /** + * Returns information about the source of this revision, or empty if the source is not know/defined + * (which is the case for command-line deployment from developers, but never for deployment jobs) + */ + public Optional<SourceRevision> source() { return source; } + + @Override + public int hashCode() { return applicationPackageHash.hashCode(); } + + @Override + public boolean equals(Object other) { + if (this == other) return true; + if ( ! (other instanceof ApplicationRevision)) return false; + return this.applicationPackageHash.equals(((ApplicationRevision)other).applicationPackageHash); + } + + @Override + public String toString() { + return "Application package revision '" + applicationPackageHash + "'" + + (source.isPresent() ? " with " + source.get() : ""); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java new file mode 100644 index 00000000000..596cbbebd45 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -0,0 +1,90 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.component.Version; + +import java.util.Objects; +import java.util.Optional; + +/** + * A change to an application + * + * @author bratseth + */ +public abstract class Change { + + /** A change to the application package revision of an application */ + public static class ApplicationChange extends Change { + + private final Optional<ApplicationRevision> revision; + + private ApplicationChange(Optional<ApplicationRevision> revision) { + Objects.requireNonNull(revision, "revision cannot be null"); + this.revision = revision; + } + + /** The revision this changes to, or empty if not known yet */ + public Optional<ApplicationRevision> revision() { return revision; } + + @Override + public int hashCode() { return revision.hashCode(); } + + @Override + public boolean equals(Object other) { + if (this == other) return true; + if ( ! (other instanceof ApplicationChange)) return false; + return ((ApplicationChange)other).revision.equals(this.revision); + } + + /** + * Creates an application change which we don't know anything about. + * We are notified that a change has occurred by completion of the component job + * but do not get to know about what the change is until a subsequent deployment + * happens. + */ + public static ApplicationChange unknown() { + return new ApplicationChange(Optional.empty()); + } + + public static ApplicationChange of(ApplicationRevision revision) { + return new ApplicationChange(Optional.of(revision)); + } + + @Override + public String toString() { + return "application change to " + revision.map(ApplicationRevision::toString).orElse("an unknown revision"); + } + + } + + /** A change to the Vespa version running an application */ + public static class VersionChange extends Change { + + private final Version version; + + public VersionChange(Version version) { + Objects.requireNonNull(version, "version cannot be null"); + this.version = version; + } + + /** The Vespa version this changes to */ + public Version version() { return version; } + + @Override + public int hashCode() { return version.hashCode(); } + + @Override + public boolean equals(Object other) { + if (this == other) return true; + if ( ! (other instanceof VersionChange)) return false; + return ((VersionChange)other).version.equals(this.version); + } + + @Override + public String toString() { + return "version change to " + version; + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java new file mode 100644 index 00000000000..75e0f82cdcf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.Zone; + +import java.time.Instant; +import java.util.Objects; + +/** + * A deployment of an application in a particular zone. + * + * @author bratseth + */ +public class Deployment { + + private final Zone zone; + private final ApplicationRevision revision; + private final Version version; + private final Instant deployTime; + + public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) { + Objects.requireNonNull(zone, "zone cannot be null"); + Objects.requireNonNull(revision, "revision cannot be null"); + Objects.requireNonNull(version, "version cannot be null"); + Objects.requireNonNull(deployTime, "deployTime cannot be null"); + this.zone = zone; + this.revision = revision; + this.version = version; + this.deployTime = deployTime; + } + + /** Returns the zone this was deployed to */ + public Zone zone() { return zone; } + + /** Returns the revision of the application which was deployed */ + public ApplicationRevision revision() { return revision; } + + /** Returns the Vespa version which was deployed */ + public Version version() { return version; } + + /** Returns the time this was deployed */ + public Instant at() { return deployTime; } + + @Override + public String toString() { + return "deployment to " + zone + " of " + revision + " on version " + version + " at " + deployTime; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java new file mode 100644 index 00000000000..d9256f94086 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -0,0 +1,333 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Controller; + +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Information about which deployment jobs an application should run and their current status. + * This is immutable. + * + * @author bratseth + */ +public class DeploymentJobs { + + private final Optional<Long> projectId; + private final ImmutableMap<JobType, JobStatus> status; + private final Optional<String> jiraIssueId; + private final boolean selfTriggering; // TODO: Remove this when no projects are self-triggering. + + /** Creates an empty set of deployment jobs */ + public DeploymentJobs(long projectId) { + this(Optional.of(projectId), ImmutableMap.of(), Optional.empty(),true); + } + + public DeploymentJobs(Optional<Long> projectId, Collection<JobStatus> jobStatusEntries, Optional<String> jiraIssueId, boolean selfTriggering) { + this(projectId, asMap(jobStatusEntries), jiraIssueId, selfTriggering); + } + + private DeploymentJobs(Optional<Long> projectId, Map<JobType, JobStatus> status, Optional<String> jiraIssueId, boolean selfTriggering) { + Objects.requireNonNull(projectId, "projectId cannot be null"); + Objects.requireNonNull(status, "status cannot be null"); + Objects.requireNonNull(jiraIssueId, "jiraIssueId cannot be null"); + this.projectId = projectId; + this.status = ImmutableMap.copyOf(status); + this.jiraIssueId = jiraIssueId; + this.selfTriggering = selfTriggering; + } + + private static Map<JobType, JobStatus> asMap(Collection<JobStatus> jobStatusEntries) { + ImmutableMap.Builder<JobType, JobStatus> b = new ImmutableMap.Builder<>(); + for (JobStatus jobStatusEntry : jobStatusEntries) + b.put(jobStatusEntry.type(), jobStatusEntry); + return b.build(); + } + + /** Return a new instance with the given completion */ + public DeploymentJobs withCompletion(JobReport report, Instant notificationTime, Controller controller) { + Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); + status.compute(report.jobType(), (type, job) -> { + if (job == null) job = JobStatus.initial(report.jobType()); + return job.withCompletion(report.jobError(), notificationTime, controller); + }); + return new DeploymentJobs(Optional.of(report.projectId()), status, jiraIssueId, report.selfTriggering()); + } + + public DeploymentJobs withTriggering(DeploymentJobs.JobType jobType, + Version version, + Optional<ApplicationRevision> revision, + Instant triggerTime) { + Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); + status.compute(jobType, (type, job) -> { + if (job == null) job = JobStatus.initial(jobType); + return job.withTriggering(version, revision, triggerTime); + }); + return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + } + + public DeploymentJobs withProjectId(long projectId) { + return new DeploymentJobs(Optional.of(projectId), status, jiraIssueId, selfTriggering); + } + + public DeploymentJobs withJiraIssueId(Optional<String> jiraIssueId) { + return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + } + + public DeploymentJobs without(JobType job) { + Map<JobType, JobStatus> status = new HashMap<>(this.status); + status.remove(job); + return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + } + + public DeploymentJobs asSelfTriggering(boolean selfTriggering) { + return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + } + + /** Returns an immutable map of the status entries in this */ + public Map<JobType, JobStatus> jobStatus() { return status; } + + /** Returns whether this application's deployment jobs trigger each other, and should be left alone, or not. */ + public boolean isSelfTriggering() { return selfTriggering; } + + /** Returns whether this has some job status which is not a success */ + public boolean hasFailures() { + return status.values().stream().anyMatch(jobStatus -> ! jobStatus.isSuccess()); + } + + /** Returns whether any job is currently in progress */ + public boolean inProgress() { + return status.values().stream().anyMatch(JobStatus::inProgress); + } + + /** Returns whether any job is failing for the given change */ + public boolean failingOn(Change change) { + return status.values().stream().anyMatch(jobStatus -> !jobStatus.isSuccess() && jobStatus.lastCompletedFor(change)); + } + + /** Returns whether change can be deployed to the given environment */ + public boolean isDeployableTo(Environment environment, Optional<Change> change) { + if (environment == null || !change.isPresent()) { + return true; + } + if (environment == Environment.staging) { + return isSuccessful(JobType.systemTest, change.get()); + } else if (environment == Environment.prod) { + return isSuccessful(JobType.stagingTest, change.get()); + } + return true; // other environments do not have any preconditions + } + + /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */ + public Instant failingSince() { + Instant failingSince = null; + for (JobStatus jobStatus : jobStatus().values()) { + if (jobStatus.isSuccess()) continue; + if (failingSince == null || failingSince.isAfter(jobStatus.firstFailing().get().at())) + failingSince = jobStatus.firstFailing().get().at(); + } + return failingSince; + } + + /** Returns the time at which the oldest running job started */ + public Optional<Instant> runningSince() { + return jobStatus().values().stream() + .filter(JobStatus::inProgress) + .sorted(Comparator.comparing(jobStatus -> jobStatus.lastTriggered().get().at())) + .map(jobStatus -> jobStatus.lastTriggered().get().at()) + .findFirst(); + } + + /** + * Returns the id of the Screwdriver project running these deployment jobs + * - or empty when this is not known or does not exist. + * It is not known until the jobs have run once and reported back to the controller. + */ + public Optional<Long> projectId() { return projectId; } + + public Optional<String> jiraIssueId() { return jiraIssueId; } + + private boolean isSuccessful(JobType jobType, Change change) { + return Optional.ofNullable(jobStatus().get(jobType)) + .filter(JobStatus::isSuccess) + .filter(status -> status.lastCompletedFor(change)) + .isPresent(); + } + + /** Job types that exist in the build system */ + public enum JobType { + + component("component"), + systemTest("system-test", zone(SystemName.cd, "test", "cd-us-central-1"), zone("test", "us-east-1")), + stagingTest("staging-test", zone(SystemName.cd, "staging", "cd-us-central-1"), zone("staging", "us-east-3")), + productionCorpUsEast1("production-corp-us-east-1", zone("prod", "corp-us-east-1")), + productionUsEast3("production-us-east-3", zone("prod", "us-east-3")), + productionUsWest1("production-us-west-1", zone("prod", "us-west-1")), + productionUsCentral1("production-us-central-1", zone("prod", "us-central-1")), + productionApNortheast1("production-ap-northeast-1", zone("prod", "ap-northeast-1")), + productionApNortheast2("production-ap-northeast-2", zone("prod", "ap-northeast-2")), + productionApSoutheast1("production-ap-southeast-1", zone("prod", "ap-southeast-1")), + productionEuWest1("production-eu-west-1", zone("prod", "eu-west-1")), + productionCdUsCentral1("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")), + productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); + + private final String id; + private final Map<SystemName, Zone> zones; + + JobType(String id, Zone... zone) { + this.id = id; + Map<SystemName, Zone> zones = new HashMap<>(); + for (Zone z : zone) { + if (zones.containsKey(z.system())) { + throw new IllegalArgumentException("A job can only map to a single zone per system"); + } + zones.put(z.system(), z); + } + this.zones = Collections.unmodifiableMap(zones); + } + + public String id() { return id; } + + /** Returns the zone for this job in the given system, or empty if this job does not have a zone */ + public Optional<Zone> zone(SystemName system) { + return Optional.ofNullable(zones.get(system)); + } + + /** Returns whether this is a production job */ + public boolean isProduction() { return environment() == Environment.prod; } + + /** Returns the environment of this job type, or null if it does not have an environment */ + public Environment environment() { + switch (this) { + case component: return null; + case systemTest: return Environment.test; + case stagingTest: return Environment.staging; + default: return Environment.prod; + } + } + + /** Returns the region of this job type, or null if it does not have a region */ + public RegionName region(SystemName system) { + return zone(system).map(Zone::region).orElse(null); + } + + public static JobType fromId(String id) { + switch (id) { + case "component" : return component; + case "system-test" : return systemTest; + case "staging-test" : return stagingTest; + case "production-corp-us-east-1" : return productionCorpUsEast1; + case "production-us-east-3" : return productionUsEast3; + case "production-us-west-1" : return productionUsWest1; + case "production-us-central-1" : return productionUsCentral1; + case "production-ap-northeast-1" : return productionApNortheast1; + case "production-ap-northeast-2" : return productionApNortheast2; + case "production-ap-southeast-1" : return productionApSoutheast1; + case "production-eu-west-1" : return productionEuWest1; + case "production-cd-us-central-1" : return productionCdUsCentral1; + case "production-cd-us-central-2" : return productionCdUsCentral2; + default : throw new IllegalArgumentException("Unknown job id '" + id + "'"); + } + } + + /** Returns the job type for the given zone, or null if none */ + public static JobType from(SystemName system, com.yahoo.config.provision.Zone zone) { + for (JobType job : values()) { + Optional<com.yahoo.config.provision.Zone> jobZone = job.zone(system); + if (jobZone.isPresent() && jobZone.get().equals(zone)) + return job; + } + return null; + } + + /** Returns the job job type for the given environment and region or null if none */ + public static JobType from(SystemName system, Environment environment, RegionName region) { + switch (environment) { + case test: return systemTest; + case staging: return stagingTest; + } + return from(system, new com.yahoo.config.provision.Zone(environment, region)); + } + + /** Returns the trigger order to use according to deployment spec */ + public static List<JobType> triggerOrder(SystemName system, DeploymentSpec deploymentSpec) { + return deploymentSpec.zones().stream() + .map(declaredZone -> JobType.from(system, declaredZone.environment(), + declaredZone.region().orElse(null))) + .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + private static Zone zone(SystemName system, String environment, String region) { + return new Zone(system, Environment.from(environment), RegionName.from(region)); + } + + private static Zone zone(String environment, String region) { + return new Zone(Environment.from(environment), RegionName.from(region)); + } + } + + /** A job report. This class is immutable. */ + public static class JobReport { + + private final ApplicationId applicationId; + private final JobType jobType; + private final long projectId; + private final long buildNumber; + private final Optional<JobError> jobError; + private final boolean selfTriggering; + private final boolean gitChanges; + + public JobReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber, Optional<JobError> jobError, boolean selfTriggering, boolean gitChanges) { + Objects.requireNonNull(applicationId, "ApplicationId can not be null."); + Objects.requireNonNull(jobType, "JobType can not be null."); + + this.applicationId = applicationId; + this.projectId = projectId; + this.jobType = jobType; + this.buildNumber = buildNumber; + this.jobError = jobError; + this.selfTriggering = selfTriggering; + this.gitChanges = gitChanges; + } + + public ApplicationId applicationId() { return applicationId; } + public JobType jobType() { return jobType; } + public long projectId() { return projectId; } + public long buildNumber() { return buildNumber; } + public boolean success() { return !jobError.isPresent(); } + public Optional<JobError> jobError() { return jobError; } + public boolean selfTriggering() { return selfTriggering; } + public boolean gitChanges() { return gitChanges; } + + } + + public enum JobError { + unknown, + outOfCapacity; + + public static Optional<JobError> from(boolean success) { + return Optional.of(success) + .filter(b -> !b) + .map(ignored -> unknown); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java new file mode 100644 index 00000000000..a30998d8517 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -0,0 +1,209 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.Controller; + +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +/** + * The last known build status of a particular deployment job for a particular application. + * This is immutable. + * + * @author bratseth + */ +public class JobStatus { + + private final DeploymentJobs.JobType type; + + private final Optional<JobRun> lastTriggered; + private final Optional<JobRun> lastCompleted; + private final Optional<JobRun> firstFailing; + private final Optional<JobRun> lastSuccess; + + private final Optional<DeploymentJobs.JobError> jobError; + + /** + * Used by the persistence layer (only) to create a complete JobStatus instance. + * Other creation should be by using initial- and with- methods. + */ + public JobStatus(DeploymentJobs.JobType type, Optional<DeploymentJobs.JobError> jobError, + Optional<JobRun> lastTriggered, Optional<JobRun> lastCompleted, + Optional<JobRun> firstFailing, Optional<JobRun> lastSuccess) { + Objects.requireNonNull(type, "jobType cannot be null"); + Objects.requireNonNull(jobError, "jobError cannot be null"); + Objects.requireNonNull(lastTriggered, "lastTriggered cannot be null"); + Objects.requireNonNull(lastCompleted, "lastCompleted cannot be null"); + Objects.requireNonNull(firstFailing, "firstFailing cannot be null"); + Objects.requireNonNull(lastSuccess, "lastSuccess cannot be null"); + + this.type = type; + this.jobError = jobError; + this.lastTriggered = lastTriggered; + this.lastCompleted = lastCompleted; + this.firstFailing = firstFailing; + this.lastSuccess = lastSuccess; + } + + /** Returns an empty job status */ + public static JobStatus initial(DeploymentJobs.JobType type) { + return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } + + public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, Instant triggerTime) { + return new JobStatus(type, jobError, Optional.of(new JobRun(version, revision, triggerTime)), + lastCompleted, firstFailing, lastSuccess); + } + + public JobStatus withCompletion(Optional<DeploymentJobs.JobError> jobError, Instant completionTime, Controller controller) { + Version version; + Optional<ApplicationRevision> revision; + if (type == DeploymentJobs.JobType.component) { // not triggered by us + version = controller.systemVersion(); + revision = Optional.empty(); + } + else if (! lastTriggered.isPresent()) { + throw new IllegalStateException("Got notified about completion of " + this + + ", but that has not been triggered nor deployed"); + + } + else { + version = lastTriggered.get().version(); + revision = lastTriggered.get().revision(); + } + + JobRun thisCompletion = new JobRun(version, revision, completionTime); + + Optional<JobRun> firstFailing = this.firstFailing; + if (jobError.isPresent() && ! this.firstFailing.isPresent()) + firstFailing = Optional.of(thisCompletion); + + Optional<JobRun> lastSuccess = this.lastSuccess; + if ( ! jobError.isPresent()) { + lastSuccess = Optional.of(thisCompletion); + firstFailing = Optional.empty(); + } + + return new JobStatus(type, jobError, lastTriggered, Optional.of(thisCompletion), firstFailing, lastSuccess); + } + + public DeploymentJobs.JobType type() { return type; } + + /** Returns true unless this job last completed with a failure */ + public boolean isSuccess() { return ! jobError.isPresent(); } + + /** The error of the last completion, or empty if the last run succeeded */ + public Optional<DeploymentJobs.JobError> jobError() { return jobError; } + + /** Returns true if job is in progress */ + public boolean inProgress() { + if (!lastTriggered().isPresent()) { + return false; + } + if (!lastCompleted().isPresent()) { + return true; + } + return lastTriggered().get().at().isAfter(lastCompleted().get().at()); + } + + /** + * Returns the last triggering of this job, or empty if the controller has never triggered it + * and not seen a deployment for it + */ + public Optional<JobRun> lastTriggered() { return lastTriggered; } + + /** Returns the last completion of this job (whether failing or succeeding), or empty if it never completed */ + public Optional<JobRun> lastCompleted() { return lastCompleted; } + + /** Returns the run when this started failing, or empty if it is not currently failing */ + public Optional<JobRun> firstFailing() { return firstFailing; } + + /** Returns the run when this last succeeded, or empty if it has never succeeded */ + public Optional<JobRun> lastSuccess() { return lastSuccess; } + + /** Returns whether the job last completed for the given change */ + public boolean lastCompletedFor(Change change) { + if (change instanceof Change.ApplicationChange) { + Change.ApplicationChange applicationChange = (Change.ApplicationChange) change; + return lastCompleted().isPresent() && lastCompleted().get().revision().equals(applicationChange.revision()); + } else if (change instanceof Change.VersionChange) { + Change.VersionChange versionChange = (Change.VersionChange) change; + return lastCompleted().isPresent() && lastCompleted().get().version().equals(versionChange.version()); + } + throw new IllegalArgumentException("Unexpected change: " + change.getClass()); + } + + @Override + public String toString() { + return "job status of " + type + "[ " + + "last triggered: " + lastTriggered.map(JobRun::toString).orElse("(never)") + + ", last completed: " + lastCompleted.map(JobRun::toString).orElse("(never)") + + ", first failing: " + firstFailing.map(JobRun::toString).orElse("(not failing)") + + ", lastSuccess: " + lastSuccess.map(JobRun::toString).orElse("(never)") + "]"; + } + + @Override + public int hashCode() { return Objects.hash(type, jobError, lastTriggered, lastCompleted, firstFailing, lastSuccess); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! ( o instanceof JobStatus)) return false; + JobStatus other = (JobStatus)o; + return Objects.equals(type, other.type) && + Objects.equals(jobError, other.jobError) && + Objects.equals(lastTriggered, other.lastTriggered) && + Objects.equals(lastCompleted, other.lastCompleted) && + Objects.equals(firstFailing, other.firstFailing) && + Objects.equals(lastSuccess, other.lastSuccess); + } + + /** Information about a particular triggering or completion of a run of a job. This is immutable. */ + public static class JobRun { + + private final Version version; + private final Optional<ApplicationRevision> revision; + private final Instant at; + + public JobRun(Version version, Optional<ApplicationRevision> revision, Instant at) { + Objects.requireNonNull(version, "version cannot be null"); + Objects.requireNonNull(revision, "revision cannot be null"); + Objects.requireNonNull(at, "at cannot be null"); + this.version = version; + this.revision = revision; + this.at = at; + } + + /** The Vespa version used on this run */ + public Version version() { return version; } + + /** The application revision used for this run, or empty when not known */ + public Optional<ApplicationRevision> revision() { return revision; } + + /** The time if this triggering or completion */ + public Instant at() { return at; } + + @Override + public int hashCode() { + return Objects.hash(version ,revision, at); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof JobRun)) return false; + JobRun other = (JobRun)o; + if ( ! Objects.equals(other.version, this.version)) return false; + if ( ! Objects.equals(this.revision, other.revision)) return false; + if ( ! Objects.equals(this.at, other.at)) return false; + return true; + } + + @Override + public String toString() { return "job run of version " + version + " " + revision + " at " + at; } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java new file mode 100644 index 00000000000..9c10e0dc153 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import java.util.Objects; + +/** + * A revision in a source repository + * + * @author bratseth + */ +public class SourceRevision { + + private final String repository; + private final String branch; + private final String commit; + + public SourceRevision(String repository, String branch, String commit) { + Objects.requireNonNull(repository, "repository cannot be null"); + Objects.requireNonNull(branch, "branch cannot be null"); + Objects.requireNonNull(commit, "commit cannot be null"); + this.repository = repository; + this.branch = branch; + this.commit = commit; + } + + public String repository() { return repository; } + public String branch() { return branch; } + public String commit() { return commit; } + + @Override + public int hashCode() { return Objects.hash(repository, branch, commit); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof SourceRevision)) return false; + + SourceRevision other = (SourceRevision)o; + return this.repository.equals(other.repository) && + this.branch.equals(other.branch) && + this.commit.equals(other.commit); + } + + @Override + public String toString() { return "source revision of repository '" + repository + + "', branch '" + branch + "' with commit '" + commit + "'"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java new file mode 100644 index 00000000000..69c846f2562 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java @@ -0,0 +1,63 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.google.common.collect.ImmutableList; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * @author bratseth + */ +public class ZipStreamReader { + + private final ImmutableList<ZipEntryWithContent> entries; + + public ZipStreamReader(InputStream input) { + try (ZipInputStream zipInput = new ZipInputStream(input)) { + ImmutableList.Builder<ZipEntryWithContent> builder = new ImmutableList.Builder<>(); + ZipEntry zipEntry; + while (null != (zipEntry = zipInput.getNextEntry())) + builder.add(new ZipEntryWithContent(zipEntry, readContent(zipInput))); + entries = builder.build(); + } + catch (IOException e) { + throw new IllegalArgumentException("IO error reading zip content", e); + } + } + + private byte[] readContent(ZipInputStream zipInput) { + try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) { + byte[] buffer = new byte[2048]; + int read; + while ( -1 != (read = zipInput.read(buffer))) + bis.write(buffer, 0, read); + return bis.toByteArray(); + } + catch (IOException e) { + throw new IllegalArgumentException("Failed reading from zipped content", e); + } + } + + public List<ZipEntryWithContent> entries() { return entries; } + + public static class ZipEntryWithContent { + + private final ZipEntry zipEntry; + private final byte[] content; + + public ZipEntryWithContent(ZipEntry zipEntry, byte[] content) { + this.zipEntry = zipEntry; + this.content = content; + } + + public ZipEntry zipEntry() { return zipEntry; } + public byte[] content() { return content; } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java new file mode 100644 index 00000000000..4dbce299b5d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Core application model + * + * @author bratseth + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java new file mode 100644 index 00000000000..df80fafd388 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.concurrent; + +import java.util.concurrent.locks.ReentrantLock; + +/** + * An acquired lock which is released on close + * + * @author bratseth + */ +public final class Lock implements AutoCloseable { + + private final ReentrantLock wrappedLock; + + Lock(ReentrantLock wrappedLock) { + this.wrappedLock = wrappedLock; + } + + /** Releases this lock */ + public void close() { + wrappedLock.unlock(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java new file mode 100644 index 00000000000..6168812203a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.concurrent; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Holds a map of locks indexed on keys of a given type. + * This is suitable in cases where exclusive access should be granted to any one of a set of keyed objects and + * there is a finite collection of keyed objects. + * + * The returned locks are reentrant (i.e the owning thread may call lock multiple times) and auto-closable. + * + * Typical use is + * <code> + * try (Lock lock = locks.lock(id)) { + * exclusive use of the object with key id + * } + * </code> + * + * @author bratseth + */ +public class Locks<TYPE> { + + private final Map<TYPE, ReentrantLock> locks = new ConcurrentHashMap<>(); + + private final long timeoutMs; + + public Locks(int timeout, TimeUnit timeoutUnit) { + timeoutMs = timeoutUnit.toMillis(timeout); + } + + /** + * Locks key. This will block until the key is acquired. + * Users of this <b>must</b> close any lock acquired. + * + * @param key the key to lock + * @return the acquired lock + * @throws TimeoutException if the lock could not be acquired within the timeout + */ + public Lock lock(TYPE key) { + try { + ReentrantLock lock = locks.computeIfAbsent(key, k -> new ReentrantLock(true)); + boolean acquired = lock.tryLock(timeoutMs, TimeUnit.MILLISECONDS); + if ( ! acquired) + throw new TimeoutException("Timed out waiting for the lock to " + key); + return new Lock(lock); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted while waiting for lock of " + key); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java new file mode 100644 index 00000000000..260761fa6ac --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.concurrent; + +/** + * Throws on timeout + * + * @author bratseth + */ +public class TimeoutException extends RuntimeException { + + public TimeoutException(String message) { + super(message); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java new file mode 100644 index 00000000000..15b3ef7fb83 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; + +import java.util.List; + +/** + * @author jvenstad + * @author mpolden + */ +public interface BuildSystem { + + /** + * Add a job for the given application to the build system + * + * @param application the application owning the job + * @param jobType the job type to be queued + * @param first whether the job should be added to the front of the queue + */ + void addJob(ApplicationId application, JobType jobType, boolean first); + + /** Remove and return a list of jobs which should be run now */ + List<BuildJob> takeJobsToRun(); + + /** Get a list of all jobs currently waiting to run */ + List<BuildJob> jobs(); + + /** Removes all queued jobs for the given application */ + void removeJobs(ApplicationId applicationId); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java new file mode 100644 index 00000000000..2bc219dde62 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -0,0 +1,368 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Responsible for scheduling deployment jobs in a build system and keeping + * Application.deploying() in sync with what is scheduled. + * + * This class is multithread safe. + * + * @author bratseth + */ +public class DeploymentTrigger { + + private final static Logger log = Logger.getLogger(DeploymentTrigger.class.getName()); + + private final Controller controller; + private final Clock clock; + private final BuildSystem buildSystem; + + public DeploymentTrigger(Controller controller, CuratorDb curator, Clock clock) { + Objects.requireNonNull(controller,"controller cannot be null"); + Objects.requireNonNull(clock,"clock cannot be null"); + this.controller = controller; + this.clock = clock; + this.buildSystem = new PolledBuildSystem(controller, curator); + } + + //--- Start of methods which triggers deployment jobs ------------------------- + + /** + * Called each time a job completes (successfully or not) to cause triggering of one or more follow-up jobs + * (which may possibly the same job once over). + * + * @param report information about the job that just completed + */ + public void triggerFromCompletion(JobReport report) { + try (Lock lock = applications().lock(report.applicationId())) { + Application application = applications().require(report.applicationId()); + application = application.withJobCompletion(report, clock.instant(), controller); + + // Handle successful first and last job + if (isFirstJob(report.jobType()) && report.success()) { // the first job tells us that a change occurred + if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) { // postpone until the current deployment is done + applications().store(application.withOutstandingChange(true), lock); + return; + } + else { // start a new change deployment + application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown())); + } + } else if (isLastJob(report.jobType(), application) && report.success()) { + application = application.withDeploying(Optional.empty()); + } + + // Trigger next + if (report.success()) + application = trigger(nextAfter(report.jobType(), application), application, report.jobType() + " completed successfully", lock); + else if (isCapacityConstrained(report.jobType()) && shouldRetryOnOutOfCapacity(application, report.jobType())) + application = trigger(report.jobType(), application, true, "Retrying due to out of capacity", lock); + else if (shouldRetryNow(application)) + application = trigger(report.jobType(), application, "Retrying as job just started failing", lock); + + applications().store(application, lock); + } + } + + /** + * Called periodically to cause triggering of jobs in the background + */ + public void triggerFailing(ApplicationId applicationId) { + try (Lock lock = applications().lock(applicationId)) { + Application application = applications().require(applicationId); + if (shouldRetryFromBeginning(application)) { + // failed for a long time: Discard existing change and restart from the component job + application = application.withDeploying(Optional.empty()); + application = trigger(JobType.component, application, "Retrying failing deployment from beginning", lock); + applications().store(application, lock); + } else { + // retry the failed job (with backoff) + for (JobType jobType : JobType.triggerOrder(controller.system(), application.deploymentSpec())) { // retry the *first* failing job + JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); + if (isFailing(jobStatus)) { + if (shouldRetryNow(jobStatus)) { + application = trigger(jobType, application, "Retrying failing job", lock); + applications().store(application, lock); + } + break; + } + } + } + } + } + + /** Triggers jobs that have been delayed according to deployment spec */ + public void triggerDelayed() { + for (Application application : applications().asList()) { + if ( ! application.deploying().isPresent() ) continue; + if (application.deploymentJobs().hasFailures()) continue; + if (application.deploymentJobs().inProgress()) continue; + + Optional<JobStatus> lastSuccessfulJob = application.deploymentJobs().jobStatus().values() + .stream() + .filter(j -> j.lastSuccess().isPresent()) + .sorted(Comparator.<JobStatus, Instant>comparing(j -> j.lastSuccess().get().at()).reversed()) + .findFirst(); + if ( ! lastSuccessfulJob.isPresent() ) continue; + + // Trigger next + try (Lock lock = applications().lock(application.id())) { + application = applications().require(application.id()); + application = trigger(nextAfter(lastSuccessfulJob.get().type(), application), application, + "Delayed by deployment spec", lock); + applications().store(application, lock); + } + } + } + + /** + * Triggers a change of this application + * + * @param applicationId the application to trigger + * @throws IllegalArgumentException if this application already have an ongoing change + */ + public void triggerChange(ApplicationId applicationId, Change change) { + try (Lock lock = applications().lock(applicationId)) { + Application application = applications().require(applicationId); + if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) + throw new IllegalArgumentException("Could not upgrade " + application + ": A change is already in progress"); + application = application.withDeploying(Optional.of(change)); + if (change instanceof Change.ApplicationChange) + application = application.withOutstandingChange(false); + application = trigger(JobType.systemTest, application, "Deploying change", lock); + applications().store(application, lock); + } + } + + /** + * Cancels any ongoing upgrade of the given application + * + * @param applicationId the application to trigger + */ + public void cancelChange(ApplicationId applicationId) { + try (Lock lock = applications().lock(applicationId)) { + Application application = applications().require(applicationId); + buildSystem.removeJobs(application.id()); + application = application.withDeploying(Optional.empty()); + applications().store(application, lock); + } + } + + //--- End of methods which triggers deployment jobs ---------------------------- + + private ApplicationController applications() { return controller.applications(); } + + /** Returns the next job to trigger after this job, or null if none should be triggered */ + private JobType nextAfter(JobType jobType, Application application) { + // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a + // new application with no previous deployments) + if (jobType == JobType.component) { + return JobType.systemTest; + } + + // At this point we've at least deployed to system test, so deployment spec should be available + List<DeploymentSpec.DeclaredZone> zones = application.deploymentSpec().zones(); + Optional<DeploymentSpec.DeclaredZone> zoneForJob = zoneForJob(application, jobType); + if (!zoneForJob.isPresent()) { + return null; + } + int zoneIndex = application.deploymentSpec().zones().indexOf(zoneForJob.get()); + + // This is last zone + if (zoneIndex == zones.size() - 1) { + return null; + } + + // Skip next job if delay has not passed yet + Duration delay = delayAfter(application, zoneForJob.get()); + Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType)) + .flatMap(JobStatus::lastSuccess) + .map(JobStatus.JobRun::at); + if (lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant())) { + log.info(String.format("Delaying next job after %s of %s by %s", jobType, application, delay)); + return null; + } + + DeploymentSpec.DeclaredZone nextZone = application.deploymentSpec().zones().get(zoneIndex + 1); + return JobType.from(controller.system(), nextZone.environment(), nextZone.region().orElse(null)); + } + + private Duration delayAfter(Application application, DeploymentSpec.DeclaredZone zone) { + int stepIndex = application.deploymentSpec().steps().indexOf(zone); + if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) { + return Duration.ZERO; + } + Duration totalDelay = Duration.ZERO; + List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps() + .subList(stepIndex + 1, application.deploymentSpec().steps().size()); + for (DeploymentSpec.Step step : remainingSteps) { + if (!(step instanceof DeploymentSpec.Delay)) { + break; + } + totalDelay = totalDelay.plus(((DeploymentSpec.Delay) step).duration()); + } + return totalDelay; + } + + private Optional<DeploymentSpec.DeclaredZone> zoneForJob(Application application, JobType jobType) { + return application.deploymentSpec() + .zones() + .stream() + .filter(z -> { + if (jobType.isProduction()) { + return z.matches(jobType.environment(), + Optional.ofNullable(jobType.region(controller.system()))); + } else { + // Ignore region for test environments as it's not specified in deployment spec + return z.environment() == jobType.environment(); + } + }) + .findFirst(); + } + + private boolean isFirstJob(JobType jobType) { + return jobType == JobType.component; + } + + private boolean isLastJob(JobType jobType, Application application) { + List<JobType> triggerOrder = JobType.triggerOrder(controller.system(), application.deploymentSpec()); + return triggerOrder.isEmpty() || jobType.equals(triggerOrder.get(triggerOrder.size() - 1)); + } + + private boolean isFailing(JobStatus jobStatusOrNull) { + return jobStatusOrNull != null && !jobStatusOrNull.isSuccess(); + } + + private boolean isCapacityConstrained(JobType jobType) { + return jobType == JobType.stagingTest || jobType == JobType.systemTest; + } + + private boolean shouldRetryFromBeginning(Application application) { + Instant eightHoursAgo = clock.instant().minus(Duration.ofHours(8)); + Instant failingSince = application.deploymentJobs().failingSince(); + if (failingSince != null && failingSince.isAfter(eightHoursAgo)) return false; + + JobStatus componentJobStatus = application.deploymentJobs().jobStatus().get(JobType.component); + if (componentJobStatus == null) return true; + if ( ! componentJobStatus.lastCompleted().isPresent() ) return true; + return componentJobStatus.lastCompleted().get().at().isBefore(eightHoursAgo); + } + + /** Decide whether the job should be triggered by the periodic trigger */ + private boolean shouldRetryNow(JobStatus job) { + if (job.isSuccess()) return false; + + if ( ! job.lastCompleted().isPresent()) return true; // Retry when we don't hear back + + // Always retry if we haven't tried in 4 hours + if (job.lastCompleted().get().at().isBefore(clock.instant().minus(Duration.ofHours(4)))) return true; + + // Wait for 10% of the time since it started failing + Duration aTenthOfFailTime = Duration.ofMillis( (clock.millis() - job.firstFailing().get().at().toEpochMilli()) / 10); + if (job.lastCompleted().get().at().isBefore(clock.instant().minus(aTenthOfFailTime))) return true; + + return false; + } + + /** Retry immediately only if this just started failing. Otherwise retry periodically */ + private boolean shouldRetryNow(Application application) { + return application.deploymentJobs().failingSince().isAfter(clock.instant().minus(Duration.ofSeconds(10))); + } + + /** Decide whether to retry due to capacity restrictions */ + private boolean shouldRetryOnOutOfCapacity(Application application, JobType jobType) { + Optional<JobError> outOfCapacityError = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType)) + .flatMap(JobStatus::jobError) + .filter(e -> e.equals(JobError.outOfCapacity)); + + if ( ! outOfCapacityError.isPresent()) return false; + + // Retry the job if it failed recently + return application.deploymentJobs().jobStatus().get(jobType).firstFailing().get().at() + .isAfter(clock.instant().minus(Duration.ofMinutes(15))); + } + + /** Decide whether job type should be triggered according to deployment spec */ + private boolean deploysTo(Application application, JobType jobType) { + Optional<Zone> zone = jobType.zone(controller.system()); + if (zone.isPresent() && jobType.isProduction()) { + // Skip triggering of jobs for zones where the application should not be deployed + if (!application.deploymentSpec().includes(jobType.environment(), Optional.of(zone.get().region()))) { + return false; + } + } + return true; + } + + /** + * Trigger a job for an application + * + * @param jobType the type of the job to trigger, or null to trigger nothing + * @param application the application to trigger the job for + * @param first whether to trigger the job before other jobs + * @param cause describes why the job is triggered + * @return the application in the triggered state, which *must* be stored by the caller + */ + private Application trigger(JobType jobType, Application application, boolean first, String cause, Lock lock) { + if (jobType == null) return application; // previous was last job + + // TODO: Remove when we can determine why this occurs + if (jobType != JobType.component && !application.deploying().isPresent()) { + log.warning(String.format("Want to trigger %s for %s with reason %s, but this application is not " + + "currently deploying a change", + jobType, application, cause)); + return application; + } + + if (!deploysTo(application, jobType)) { + return application; + } + + if (!application.deploymentJobs().isDeployableTo(jobType.environment(), application.deploying())) { + log.warning(String.format("Want to trigger %s for %s with reason %s, but change is untested", jobType, + application, cause)); + return application; + } + + if (application.deploymentJobs().isSelfTriggering()) { + log.info("Not triggering " + jobType + " for self-triggering " + application); + return application; + } + + log.info(String.format("Triggering %s for %s, %s: %s", jobType, application, + application.deploying().map(d -> "deploying " + d).orElse("restarted deployment"), + cause)); + buildSystem.addJob(application.id(), jobType, first); + + return application.withJobTriggering(jobType, clock.instant(), controller); + } + + private Application trigger(JobType jobType, Application application, String cause, Lock lock) { + return trigger(jobType, application, false, cause, lock); + } + + public BuildSystem buildSystem() { return buildSystem; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java new file mode 100644 index 00000000000..41adb4abe6a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java @@ -0,0 +1,100 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.List; + +/** + * Stores a queue for each type of job, and offers jobs from each of these to a periodic + * polling mechanism which is responsible for triggering the offered jobs in an external build service. + * + * @author jvenstad + * @author mpolden + */ +public class PolledBuildSystem implements BuildSystem { + + private final Controller controller; + + private final CuratorDb curator; + + public PolledBuildSystem(Controller controller, CuratorDb curator) { + this.controller = controller; + this.curator = curator; + } + + @Override + public void addJob(ApplicationId application, JobType jobType, boolean first) { + try (Lock lock = curator.lockJobQueues()) { + Deque<ApplicationId> queue = curator.readJobQueue(jobType); + if ( ! queue.contains(application)) { + if (first) { + queue.addFirst(application); + } else { + queue.add(application); + } + } + curator.writeJobQueue(jobType, queue); + } + } + + @Override + public List<BuildJob> jobs() { + return getJobs(false); + } + + @Override + public List<BuildJob> takeJobsToRun() { + return getJobs(true); + } + + + @Override + public void removeJobs(ApplicationId application) { + try (Lock lock = curator.lockJobQueues()) { + for (JobType jobType : JobType.values()) { + Deque<ApplicationId> queue = curator.readJobQueue(jobType); + while (queue.remove(application)) { + // keep removing until not found + } + curator.writeJobQueue(jobType, queue); + } + } + } + + private List<BuildJob> getJobs(boolean removeFromQueue) { + try (Lock lock = curator.lockJobQueues()) { + List<BuildJob> jobsToRun = new ArrayList<>(); + for (JobType jobType : JobType.values()) { + Deque<ApplicationId> queue = curator.readJobQueue(jobType); + for (ApplicationId a : queue) { + ApplicationId application = removeFromQueue ? queue.poll() : a; + jobsToRun.add(new BuildJob(projectIdFor(application), jobType.id())); + + // Return only one job at a time for capacity constrained queues + if (removeFromQueue && isCapacityConstrained(jobType)) break; + } + if (removeFromQueue) + curator.writeJobQueue(jobType, queue); + } + return Collections.unmodifiableList(jobsToRun); + } + } + + private Long projectIdFor(ApplicationId applicationId) { + return controller.applications().require(applicationId).deploymentJobs().projectId().get(); + } + + private static boolean isCapacityConstrained(JobType jobType) { + return jobType == JobType.stagingTest || jobType == JobType.systemTest; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java new file mode 100644 index 00000000000..016ea66cb1a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -0,0 +1,67 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.Contacts; +import com.yahoo.vespa.hosted.controller.api.integration.Issues; +import com.yahoo.vespa.hosted.controller.api.integration.Properties; +import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; + +import java.time.Duration; + +/** + * Maintenance jobs of the controller. + * Each maintenance job is a singleton instance of its implementing class, created and owned by this, + * and running its own dedicated thread. + * + * @author bratseth + */ +public class ControllerMaintenance extends AbstractComponent { + + private final JobControl jobControl; + + private final DeploymentExpirer deploymentExpirer; + private final DeploymentIssueReporter deploymentIssueReporter; + private final MetricsReporter metricsReporter; + private final FailureRedeployer failureRedeployer; + private final OutstandingChangeDeployer outstandingChangeDeployer; + private final VersionStatusUpdater versionStatusUpdater; + private final Upgrader upgrader; + private final DelayedDeployer delayedDeployer; + + @SuppressWarnings("unused") // instantiated by Dependency Injection + public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, + JobControl jobControl, Metric metric, Chef chefClient, + Contacts contactsClient, Properties propertiesClient, Issues issuesClient) { + Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); + this.jobControl = jobControl; + deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); + deploymentIssueReporter = new DeploymentIssueReporter(controller, contactsClient, propertiesClient, + issuesClient, maintenanceInterval, jobControl); + metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); + failureRedeployer = new FailureRedeployer(controller, maintenanceInterval, jobControl); + outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl); + versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl); + upgrader = new Upgrader(controller, maintenanceInterval, jobControl); + delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl); + } + + /** Returns control of the maintenance jobs of this */ + public JobControl jobControl() { return jobControl; } + + @Override + public void deconstruct() { + deploymentExpirer.deconstruct(); + deploymentIssueReporter.deconstruct(); + metricsReporter.deconstruct(); + failureRedeployer.deconstruct(); + outstandingChangeDeployer.deconstruct(); + versionStatusUpdater.deconstruct(); + upgrader.deconstruct(); + delayedDeployer.deconstruct(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java new file mode 100644 index 00000000000..cb09c41a034 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Controller; + +import java.time.Duration; + +/** + * Maintenance job which triggers jobs that have been delayed according to the applications deployment spec. + * + * @author mpolden + */ +public class DelayedDeployer extends Maintainer { + + public DelayedDeployer(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + @Override + protected void maintain() { + controller().applications().deploymentTrigger().triggerDelayed(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java new file mode 100644 index 00000000000..eb44229e790 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.application.Deployment; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.logging.Level; + +/** + * Expires instances in zones that have configured expiration using TimeToLive. + * + * @author mortent + * @author bratseth + */ +public class DeploymentExpirer extends Maintainer { + + private final Clock clock; + + public DeploymentExpirer(Controller controller, Duration interval, JobControl jobControl) { + this(controller, interval, Clock.systemUTC(), jobControl); + } + + public DeploymentExpirer(Controller controller, Duration interval, Clock clock, JobControl jobControl) { + super(controller, interval, jobControl); + this.clock = clock; + } + + @Override + protected void maintain() { + for (Application application : controller().applications().asList()) { + for (Deployment deployment : application.deployments().values()) { + if (deployment.zone().environment().equals(Environment.prod)) continue; + + if (hasExpired(controller().zoneRegistry(), deployment, clock.instant())) + deactivate(application, deployment); + } + } + } + + private void deactivate(Application application, Deployment deployment) { + try { + controller().applications().deactivate(application, deployment, true); + } + catch (Exception e) { + log.log(Level.WARNING, "Could not expire " + deployment + " of " + application, e); + } + } + + public static boolean hasExpired(ZoneRegistry zoneRegistry, Deployment deployment, Instant now) { + return zoneRegistry.getDeploymentTimeToLive(deployment.zone().environment(), deployment.zone().region()) + .map(duration -> getExpiration(deployment, duration)) + .map(now::isAfter) + .orElse(false); + } + + private static Instant getExpiration(Deployment instance, Duration ttl) { + return instance.at().plus(ttl); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java new file mode 100644 index 00000000000..90544a8ac30 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -0,0 +1,234 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.Contacts; +import com.yahoo.vespa.hosted.controller.api.integration.Contacts.UserContact; +import com.yahoo.vespa.hosted.controller.api.integration.Issues; +import com.yahoo.vespa.hosted.controller.api.integration.Issues.Classification; +import com.yahoo.vespa.hosted.controller.api.integration.Issues.Issue; +import com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo; +import com.yahoo.vespa.hosted.controller.api.integration.Properties; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.admin; +import static com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo.Status.done; + +/** + * Maintenance job which creates Jira issues for tenants when they have jobs which fails continuously + * and escalates issues which are not handled. + * + * @author jvenstad + */ +public class DeploymentIssueReporter extends Maintainer { + + static final Duration maxFailureAge = Duration.ofDays(2); + static final Duration maxInactivityAge = Duration.ofDays(4); + static final String deploymentFailureLabel = "vespaDeploymentFailure"; + static final Classification vespaOps = new Classification("VESPA", "Services", deploymentFailureLabel); + static final UserContact terminalUser = new UserContact("frodelu", "Frode Lundgren", admin); + + private final Contacts contacts; + private final Properties properties; + private final Issues issues; + + DeploymentIssueReporter(Controller controller, Contacts contacts, Properties properties, Issues issues, + Duration maintenanceInterval, JobControl jobControl) { + super(controller, maintenanceInterval, jobControl); + this.contacts = contacts; + this.properties = properties; + this.issues = issues; + } + + @Override + protected void maintain() { + maintainDeploymentIssues(controller().applications().asList()); + escalateInactiveDeploymentIssues(controller().applications().asList()); + } + + /** + * File issues for applications which have failed deployment for longer than @maxFailureAge + * and store the issue id for the filed issues. Also, clear the @issueIds of applications + * where deployment has not failed for this amount of time. + */ + private void maintainDeploymentIssues(List<Application> applications) { + Collection<Application> failingApplications = new ArrayList<>(); + for (Application application : applications) + if (failingSinceBefore(application.deploymentJobs(), controller().clock().instant().minus(maxFailureAge))) + failingApplications.add(application); + else + controller().applications().setJiraIssueId(application.id(), Optional.empty()); + + // TODO: Do this when version.confidence is BROKEN instead? + if (failingApplications.size() > 0.2 * applications.size()) { + fileOrUpdate(manyFailingDeploymentsIssueFrom(failingApplications)); // Problems with Vespa is the most likely cause when so many deployments fail. + } + else { + for (Application application : failingApplications) { + Issue deploymentIssue = deploymentIssueFrom(application); + Classification applicationOwner = null; + try { + applicationOwner = jiraClassificationOf(ownerOf(application)); + fileFor(application, deploymentIssue.with(applicationOwner)); + } + catch (RuntimeException e) { // Catch errors due to inconsistent or missing data in Sherpa, OpsDB, JIRA, and send to ourselves. + Pattern componentError = Pattern.compile(".*Component name '.*' is not valid.*", Pattern.DOTALL); + if (componentError.matcher(e.getMessage()).matches()) // Several properties seem to list invalid components, in which case we simply ignore this. + fileFor(application, deploymentIssue.with(applicationOwner.withComponent(null))); + else + fileFor(application, deploymentIssue.append(e.getMessage() + "\n\nAddressee:\n" + applicationOwner)); + } + } + } + } + + /** Returns whether @deploymentJobs has a job which has been failing since before @failureThreshold or not. */ + private boolean failingSinceBefore(DeploymentJobs deploymentJobs, Instant failureThreshold) { + return deploymentJobs.hasFailures() && deploymentJobs.failingSince().isBefore(failureThreshold); + } + + private Tenant ownerOf(Application application) { + return controller().tenants().tenant(new TenantId(application.id().tenant().value())).get(); + } + + /** Use the @propertyId of @tenant, if present, to look up JIRA information in OpsDB. */ + private Classification jiraClassificationOf(Tenant tenant) { + Long propertyId = tenant.getPropertyId().map(PropertyId::value).orElseThrow(() -> + new NoSuchElementException("No property id is listed for " + tenant)); + + Classification classification = properties.classificationFor(propertyId).orElseThrow(() -> + new NoSuchElementException("No property was found with id " + propertyId)); + + return classification.withLabel(deploymentFailureLabel); + } + + /** File @issue for @application, if @application doesn't already have an @Issue associated with it. */ + private void fileFor(Application application, Issue issue) { + Optional<String> ourIssueId = application.deploymentJobs().jiraIssueId() + .filter(jiraIssueId -> issues.fetch(jiraIssueId).status() != done); + + if ( ! ourIssueId.isPresent()) + controller().applications().setJiraIssueId(application.id(), Optional.of(issues.file(issue))); + } + + /** File @issue, or update a JIRA issue representing the same issue. */ + private void fileOrUpdate(Issue issue) { + Optional<String> jiraIssueId = issues.fetchSimilarTo(issue) + .stream().findFirst().map(Issues.IssueInfo::id); + + if (jiraIssueId.isPresent()) + issues.update(jiraIssueId.get(), issue.description()); + else + issues.file(issue); + } + + /** Escalate JIRA issues for which there has been no activity for a set amount of time. */ + private void escalateInactiveDeploymentIssues(List<Application> applications) { + applications.forEach(application -> + application.deploymentJobs().jiraIssueId().ifPresent(jiraIssueId -> { + Issues.IssueInfo issueInfo = issues.fetch(jiraIssueId); + if (issueInfo.updated().isBefore(controller().clock().instant().minus(maxInactivityAge))) + escalateAndComment(issueInfo, application); + })); + } + + /** Reassign the JIRA issue for @application one step up in the OpsDb escalation chain, and add an explanatory comment to it. */ + private void escalateAndComment(IssueInfo issueInfo, Application application) { + Optional<String> assignee = issueInfo.assignee(); + if (assignee.isPresent()) { + if (assignee.get().equals(terminalUser.username())) return; + issues.addWatcher(issueInfo.id(), assignee.get()); + } + + Long propertyId = ownerOf(application).getPropertyId().get().value(); + + UserContact escalationTarget = contacts.escalationTargetFor(propertyId, assignee.orElse("no one")); + if (escalationTarget.is(assignee.orElse("no one"))) + escalationTarget = terminalUser; + + String comment = deploymentIssueEscalationComment(application, propertyId, assignee.orElse("anyone")); + + issues.comment(issueInfo.id(), comment); + issues.reassign(issueInfo.id(), escalationTarget.username()); + } + + Issue deploymentIssueFrom(Application application) { + return new Issue(deploymentIssueSummary(application), deploymentIssueDescription(application)) + .with(vespaOps); + } + + Issue manyFailingDeploymentsIssueFrom(Collection<Application> applications) { + return new Issue( + "More than 20% of Hosted Vespa deployments are failing", + applications.stream() + .map(application -> "[" + application.id().toShortString() + "|" + toUrl(application.id()) + "]") + .collect(Collectors.joining("\n")), + vespaOps); + } + + // TODO: Use the method of the same name in ApplicationId + private static String toShortString(ApplicationId id) { + return id.tenant().value() + "." + id.application().value() + + ( id.instance().isDefault() ? "" : "." + id.instance().value() ); + } + + private String toUrl(ApplicationId applicationId) { + return controller().zoneRegistry().getDashboardUri().resolve("/apps" + + "/tenant/" + applicationId.tenant().value() + + "/application/" + applicationId.application().value()).toString(); + } + + private String toOpsDbUrl(long propertyId) { + return contacts.contactsUri(propertyId).toString(); + + } + + /** Returns the summary text what will be assigned to a new issue */ + private static String deploymentIssueSummary(Application application) { + return "[" + toShortString(application.id()) + "] Action required: Repair deployment"; + } + + /** Returns the description text what will be assigned to a new issue */ + private String deploymentIssueDescription(Application application) { + return "Deployment jobs of the Vespa application " + + "[" + toShortString(application.id()) + "|" + toUrl(application.id()) + "] have been failing " + + "continuously for over 48 hours. This blocks any change to this application from being deployed " + + "and will also block global rollout of new Vespa versions for everybody.\n\n" + + "Please assign your highest priority to fixing this. If you need support, request it using " + + "[yo/vespa-support|http://yo/vespa-support]. " + + "If this application is not in use, please re-assign this issue to project \"VESPA\" " + + "with component \"Services\", and ask for the application to be removed.\n\n" + + "If we do not get a response on this issue, we will auto-escalate it."; + } + + /** Returns the comment text that what will be added to an issue each time it is escalated */ + private String deploymentIssueEscalationComment(Application application, long propertyId, String priorAssignee) { + return "This issue tracks the failing deployment of Vespa application " + + "[" + toShortString(application.id()) + "|" + toUrl(application.id()) + "]. " + + "Since we have not received a response from " + priorAssignee + + ", we are escalating to you, " + + "based on [your OpsDb information|" + toOpsDbUrl(propertyId) + "]. " + + "Please acknowledge this issue and assign somebody to " + + "fix it as soon as possible.\n\n" + + "If we do not receive a response we will keep auto-escalating this issue. " + + "If we run out of escalation options for your OpsDb property, we will assume this application " + + "is not managed by anyone and DELETE it. In the meantime, this issue will block global deployment " + + "of Vespa for the entire company."; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java new file mode 100644 index 00000000000..9e8f902a8db --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +/** + * Attempts redeployment of failed jobs and deployments. + * + * @author bratseth + */ +public class FailureRedeployer extends Maintainer { + + public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + @Override + public void maintain() { + ApplicationList applications = ApplicationList.from(controller().applications().asList()).isDeploying(); + List<Application> toTrigger = new ArrayList<>(); + + // Applications with deployment failures for current change and no running jobs + toTrigger.addAll(applications.hasDeploymentFailures() + .notRunningJob() + .asList()); + + // Applications with jobs that have been in progress for more than 12 hours + Instant twelveHoursAgo = controller().clock().instant().minus(Duration.ofHours(12)); + toTrigger.addAll(applications.jobRunningSince(twelveHoursAgo).asList()); + + toTrigger.forEach(application -> controller().applications().deploymentTrigger() + .triggerFailing(application.id())); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java new file mode 100644 index 00000000000..e05612aaf57 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java @@ -0,0 +1,67 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.logging.Logger; + +/** + * Provides status and control over running maintenance jobs. + * This is multithread safe. + * + * Job deactivation is stored in a local file. + * + * @author bratseth + */ +public class JobControl { + + private static final Logger log = Logger.getLogger(JobControl.class.getName()); + + private final CuratorDb curator; + + /** This is not stored in ZooKeeper as all nodes start all jobs */ + private final Set<String> startedJobs = new ConcurrentSkipListSet<>(); + + /** Create a job control instance which persists activation changes to the default directory */ + public JobControl(CuratorDb curator) { + this.curator = curator; + } + + public CuratorDb curator() { return curator; } + + /** Notifies this that a job was started */ + public void started(String jobSimpleClassName) { + startedJobs.add(jobSimpleClassName); + } + + /** + * Returns a snapshot of the set of jobs started on this system (whether deactivated or not). + * Each job is represented by its simple (omitting package) class name. + */ + public Set<String> jobs() { return new HashSet<>(startedJobs); } + + /** Returns an unmodifiable set containing the currently inactive jobs in this */ + public Set<String> inactiveJobs() { return curator.readInactiveJobs(); } + + /** Returns true if this job is not currently deactivated */ + public boolean isActive(String jobSimpleClassName) { + return ! inactiveJobs().contains(jobSimpleClassName); + } + + /** Set a job active or inactive */ + public void setActive(String jobSimpleClassName, boolean active) { + try (Lock lock = curator.lockInactiveJobs()) { + Set<String> inactiveJobs = curator.readInactiveJobs(); + if (active) + inactiveJobs.remove(jobSimpleClassName); + else + inactiveJobs.add(jobSimpleClassName); + curator.writeInactiveJobs(inactiveJobs); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java new file mode 100644 index 00000000000..9f9f0175230 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java @@ -0,0 +1,80 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller. + * + * @author bratseth + */ +public abstract class Maintainer extends AbstractComponent implements Runnable { + + protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); + + private final Controller controller; + private final Duration maintenanceInterval; + private final JobControl jobControl; + private final ScheduledExecutorService service; + + public Maintainer(Controller controller, Duration interval, JobControl jobControl) { + this.controller = controller; + this.maintenanceInterval = interval; + this.jobControl = jobControl; + + service = new ScheduledThreadPoolExecutor(1); + service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); + jobControl.started(name()); + } + + protected Controller controller() { return controller; } + + protected CuratorDb curator() { return jobControl.curator(); } + + @Override + public void run() { + try { + if (jobControl.isActive(name())) { + try (Lock lock = jobControl.curator().lockMaintenanceJob(name())) { + maintain(); + } + } + } + catch (UncheckedTimeoutException e) { + // another controller instance is running this job at the moment; ok + } + catch (RuntimeException e) { + log.log(Level.WARNING, this + " failed. Will retry in " + maintenanceInterval.toMinutes() + " minutes", e); + } + } + + @Override + public void deconstruct() { + this.service.shutdown(); + } + + /** Called once each time this maintenance job should run */ + protected abstract void maintain(); + + public Duration maintenanceInterval() { return maintenanceInterval; } + + public String name() { return this.getClass().getSimpleName(); } + + /** Returns the name of this */ + @Override + public final String toString() { + return name(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java new file mode 100644 index 00000000000..3d0cd284c55 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -0,0 +1,118 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; +import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * @author mortent + */ +public class MetricsReporter extends Maintainer { + + public static final String convergeMetric = "seconds.since.last.chef.convergence"; + public static final String deploymentFailMetric = "deployment.failurePercentage"; + private final Metric metric; + private final Chef chefClient; + private final Clock clock; + private final SystemName system; + + public MetricsReporter(Controller controller, Metric metric, Chef chefClient, JobControl jobControl, + SystemName system) { + this(controller, metric, chefClient, Clock.systemUTC(), jobControl, system); + } + + public MetricsReporter(Controller controller, Metric metric, Chef chefClient, Clock clock, + JobControl jobControl, SystemName system) { + super(controller, Duration.ofMinutes(1), jobControl); // use fixed rate for metrics + this.metric = metric; + this.chefClient = chefClient; + this.clock = clock; + this.system = system; + } + + @Override + public void maintain() { + reportChefMetrics(); + reportDeploymentMetrics(); + } + + private void reportChefMetrics() { + String query = "chef_environment:hosted*"; + if (system == SystemName.cd) { + query += " AND hosted_system:" + system; + } + PartialNodeResult nodeResult = chefClient.partialSearchNodes(query, + Arrays.asList( + AttributeMapping.simpleMapping("fqdn"), + AttributeMapping.simpleMapping("ohai_time"), + AttributeMapping.deepMapping("tenant", Arrays.asList("hosted", "owner", "tenant")), + AttributeMapping.deepMapping("application", Arrays.asList("hosted", "owner", "application")), + AttributeMapping.deepMapping("instance", Arrays.asList("hosted", "owner", "instance")), + AttributeMapping.deepMapping("environment", Arrays.asList("hosted", "environment")), + AttributeMapping.deepMapping("region", Arrays.asList("hosted", "region")), + AttributeMapping.deepMapping("system", Arrays.asList("hosted", "system")) + )); + + // The above search will return a correct list if the system is CD. However for main, it will + // return all nodes, since system==nil for main + keepNodesWithSystem(nodeResult, system); + + Instant instant = clock.instant(); + for (PartialNode node : nodeResult.rows) { + String hostname = node.getFqdn(); + long secondsSinceConverge = Duration.between(Instant.ofEpochSecond(node.getOhaiTime().longValue()), instant).getSeconds(); + Map<String, String> dimensions = new HashMap<>(); + dimensions.put("host", hostname); + dimensions.put("system", node.getValue("system").orElse("main")); + Optional<String> environment = node.getValue("environment"); + Optional<String> region = node.getValue("region"); + + if(environment.isPresent() && region.isPresent()) { + dimensions.put("zone", String.format("%s.%s", environment.get(), region.get())); + } + + node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant)); + Optional<String> application = node.getValue("application"); + if (application.isPresent()) { + dimensions.put("app",String.format("%s.%s", application.get(), node.getValue("instance").orElse("default"))); + } + Metric.Context context = metric.createContext(dimensions); + metric.set(convergeMetric, secondsSinceConverge, context); + } + } + + private void reportDeploymentMetrics() { + metric.set(deploymentFailMetric, deploymentFailRatio() * 100, metric.createContext(Collections.emptyMap())); + } + + private double deploymentFailRatio() { + List<Application> applications = controller().applications().asList(); + if (applications.isEmpty()) return 0; + + return (double)applications.stream().filter(a -> a.deploymentJobs().hasFailures()).count() / + (double)applications.size(); + } + + private void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) { + nodeResult.rows.removeIf(node -> !system.name().equals(node.getValue("system").orElse("main"))); + } + +} + + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java new file mode 100644 index 00000000000..4485a603f61 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.Change; + +import java.time.Duration; + +/** + * Deploys application changes which have been postponed due to an ongoing upgrade + * + * @author bratseth + */ +public class OutstandingChangeDeployer extends Maintainer { + + public OutstandingChangeDeployer(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + @Override + protected void maintain() { + ApplicationList applications = ApplicationList.from(controller().applications().asList()).notPullRequest(); + for (Application application : applications.asList()) { + if (application.hasOutstandingChange() && ! application.deploying().isPresent()) + controller().applications().deploymentTrigger().triggerChange(application.id(), + Change.ApplicationChange.unknown()); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java new file mode 100644 index 00000000000..b3d75106d2f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -0,0 +1,94 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; +import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import com.yahoo.yolean.Exceptions; + +import java.time.Duration; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Maintenance job which schedules applications for Vespa version upgrade + * + * @author bratseth + */ +public class Upgrader extends Maintainer { + + private static final Logger log = Logger.getLogger(Upgrader.class.getName()); + + public Upgrader(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + /** + * Schedule application upgrades. Note that this implementation must be idempotent. + */ + @Override + public void maintain() { + VespaVersion target = controller().versionStatus().version(controller().systemVersion()); + if (target == null) return; // we don't have information about the current system version at this time + + // TODO: Remove corp-prod special casing when corp-prod and main are upgraded at the same time + if (Vtag.currentVersion.isAfter(target.versionNumber())) { + upgrade(applications().deploysTo(Environment.prod, RegionName.from("corp-us-east-1")).with(UpgradePolicy.canary), + Vtag.currentVersion); + } + + switch (target.confidence()) { + case broken: + log.info(String.format("Version %s is broken, cancelling all upgrades", target.versionNumber())); + cancelUpgradesOf(applications().upgradingTo(target.versionNumber()) + .without(UpgradePolicy.canary)); // keep trying canaries + break; + case low: + upgrade(applications().with(UpgradePolicy.canary), target.versionNumber()); + break; + case normal: + upgrade(applications().with(UpgradePolicy.defaultPolicy), target.versionNumber()); + break; + case high: + upgrade(applications().with(UpgradePolicy.conservative), target.versionNumber()); + break; + default: + throw new IllegalArgumentException("Unknown version confidence " + target.confidence()); + } + } + + /** Returns a list of all applications */ + private ApplicationList applications() { return ApplicationList.from(controller().applications().asList()); } + + private void upgrade(ApplicationList applications, Version version) { + Change.VersionChange change = new Change.VersionChange(version); + cancelUpgradesOf(applications.upgradingToLowerThan(version)); + applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade + applications = applications.onLowerVersionThan(version); + applications = applications.notDeployingApplication(); // wait with applications deploying an application change + applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version + applications = applications.notRunningJobFor(change); // do not trigger multiple jobs simultaneously for same upgrade + for (Application application : applications.byIncreasingDeployedVersion().asList()) { + try { + controller().applications().deploymentTrigger().triggerChange(application.id(), change); + } catch (IllegalArgumentException e) { + log.log(Level.INFO, "Could not trigger change: " + Exceptions.toMessageString(e)); + } + } + } + + private void cancelUpgradesOf(ApplicationList applications) { + for (Application application : applications.asList()) { + controller().applications().deploymentTrigger().cancelChange(application.id()); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java new file mode 100644 index 00000000000..dea991bc653 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; + +import java.io.UncheckedIOException; +import java.time.Duration; + +/** + * This maintenance job periodically updates the version status. + * Since the version status is expensive to compute and do not need to be perfectly up to date, + * we do not want to recompute it each time it is accessed. + * + * @author bratseth + */ +public class VersionStatusUpdater extends Maintainer { + + public VersionStatusUpdater(Controller controller, Duration interval, JobControl jobControl) { + super(controller, interval, jobControl); + } + + @Override + protected void maintain() { + try { + VersionStatus newStatus = VersionStatus.compute(controller()); + controller().updateVersionStatus(newStatus); + } catch (UncheckedIOException e) { + log.warning("Failed to compute version status. This is likely a transient error: " + e.getMessage()); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java new file mode 100644 index 00000000000..14267807041 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java new file mode 100644 index 00000000000..112e90e2cd7 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * The root package of the controller + * + * @author bratseth + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java new file mode 100644 index 00000000000..014c63a6779 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -0,0 +1,304 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +/** + * Serializes applications to/from slime. + * This class is multithread safe. + * + * @author bratseth + */ +public class ApplicationSerializer { + + // Application fields + private final String idField = "id"; + private final String deploymentSpecField = "deploymentSpecField"; + private final String validationOverridesField = "validationOverrides"; + private final String deploymentsField = "deployments"; + private final String deploymentJobsField = "deploymentJobs"; + private final String deployingField = "deployingField"; + private final String outstandingChangeField = "outstandingChangeField"; + + // Deployment fields + private final String zoneField = "zone"; + private final String environmentField = "environment"; + private final String regionField = "region"; + private final String deployTimeField = "deployTime"; + private final String applicationPackageRevisionField = "applicationPackageRevision"; + private final String applicationPackageHashField = "applicationPackageHash"; + private final String sourceRevisionField = "sourceRevision"; + private final String repositoryField = "repositoryField"; + private final String branchField = "branchField"; + private final String commitField = "commitField"; + + // DeploymentJobs fields + private final String projectIdField = "projectId"; + private final String jobStatusField = "jobStatus"; + private final String jiraIssueIdField = "jiraIssueId"; + private final String selfTriggeringField = "selfTriggering"; + + // JobStatus field + private final String jobTypeField = "jobType"; + private final String errorField = "jobError"; + private final String completionTimeField = "completionTime"; + private final String failingSinceField = "failingSince"; + private final String lastTriggeredField = "lastTriggered"; + private final String lastCompletedField = "lastCompleted"; + private final String firstFailingField = "firstFailing"; + private final String lastSuccessField = "lastSuccess"; + + // JobRun fields + private final String versionField = "version"; + private final String revisionField = "revision"; + private final String atField = "at"; + + // ------------------ Serialization + + public Slime toSlime(Application application) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString(idField, application.id().serializedForm()); + root.setString(deploymentSpecField, application.deploymentSpec().xmlForm()); + root.setString(validationOverridesField, application.validationOverrides().xmlForm()); + deploymentsToSlime(application.deployments().values(), root.setArray(deploymentsField)); + toSlime(application.deploymentJobs(), root.setObject(deploymentJobsField)); + toSlime(application.deploying(), root); + root.setBool(outstandingChangeField, application.hasOutstandingChange()); + return slime; + } + + private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) { + for (Deployment deployment : deployments) + deploymentToSlime(deployment, array.addObject()); + } + + private void deploymentToSlime(Deployment deployment, Cursor object) { + zoneToSlime(deployment.zone(), object.setObject(zoneField)); + object.setString(versionField, deployment.version().toString()); + object.setLong(deployTimeField, deployment.at().toEpochMilli()); + toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); + } + + private void zoneToSlime(Zone zone, Cursor object) { + object.setString(environmentField, zone.environment().value()); + object.setString(regionField, zone.region().value()); + } + + private void toSlime(ApplicationRevision applicationRevision, Cursor object) { + object.setString(applicationPackageHashField, applicationRevision.id()); + if (applicationRevision.source().isPresent()) + toSlime(applicationRevision.source().get(), object.setObject(sourceRevisionField)); + } + + private void toSlime(SourceRevision sourceRevision, Cursor object) { + object.setString(repositoryField, sourceRevision.repository()); + object.setString(branchField, sourceRevision.branch()); + object.setString(commitField, sourceRevision.commit()); + } + + private void toSlime(DeploymentJobs deploymentJobs, Cursor cursor) { + deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); + jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); + deploymentJobs.jiraIssueId().ifPresent(jiraIssueId -> cursor.setString(jiraIssueIdField, jiraIssueId)); + cursor.setBool(selfTriggeringField, deploymentJobs.isSelfTriggering()); + } + + private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) { + for (JobStatus jobStatus : jobStatuses) + toSlime(jobStatus, jobStatusArray.addObject()); + } + + private void toSlime(JobStatus jobStatus, Cursor object) { + object.setString(jobTypeField, jobStatus.type().id()); + if (jobStatus.jobError().isPresent()) + object.setString(errorField, jobStatus.jobError().get().name()); + + jobRunToSlime(jobStatus.lastTriggered(), object, lastTriggeredField); + jobRunToSlime(jobStatus.lastCompleted(), object, lastCompletedField); + jobRunToSlime(jobStatus.firstFailing(), object, firstFailingField); + jobRunToSlime(jobStatus.lastSuccess(), object, lastSuccessField); + } + + private void jobRunToSlime(Optional<JobStatus.JobRun> jobRun, Cursor parent, String jobRunObjectName) { + if ( ! jobRun.isPresent()) return; + Cursor object = parent.setObject(jobRunObjectName); + object.setString(versionField, jobRun.get().version().toString()); + if ( jobRun.get().revision().isPresent()) + toSlime(jobRun.get().revision().get(), object.setObject(revisionField)); + object.setLong(atField, jobRun.get().at().toEpochMilli()); + } + + private void toSlime(Optional<Change> deploying, Cursor parentObject) { + if ( ! deploying.isPresent()) return; + + Cursor object = parentObject.setObject(deployingField); + if (deploying.get() instanceof Change.VersionChange) + object.setString(versionField, ((Change.VersionChange)deploying.get()).version().toString()); + else if (((Change.ApplicationChange)deploying.get()).revision().isPresent()) + toSlime(((Change.ApplicationChange)deploying.get()).revision().get(), object); + } + + // ------------------ Deserialization + + public Application fromSlime(Slime slime) { + Inspector root = slime.get(); + + ApplicationId id = ApplicationId.fromSerializedForm(root.field(idField).asString()); + DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString()); + ValidationOverrides validationOverrides = validationOverridesFromSlime(root.field(validationOverridesField)); + List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField)); + DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField)); + Optional<Change> deploying = changeFromSlime(root.field(deployingField)); + boolean outstandingChange = root.field(outstandingChangeField).asBool(); + + return new Application(id, deploymentSpec, validationOverrides, deployments, + deploymentJobs, deploying, outstandingChange); + } + + private ValidationOverrides validationOverridesFromSlime(Inspector field) { + if ( ! field.valid()) return ValidationOverrides.empty; // TODO: Remove this line (and inline function) after June 2017 + return ValidationOverrides.fromXml(field.asString()); + } + + private List<Deployment> deploymentsFromSlime(Inspector array) { + List<Deployment> deployments = new ArrayList<>(); + array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item))); + return deployments; + } + + private Deployment deploymentFromSlime(Inspector deploymentObject) { + return new Deployment(zoneFromSlime(deploymentObject.field(zoneField)), + applicationRevisionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(), + Version.fromString(deploymentObject.field(versionField).asString()), + Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong())); + } + + private Zone zoneFromSlime(Inspector object) { + return new Zone(Environment.from(object.field(environmentField).asString()), + RegionName.from(object.field(regionField).asString())); + } + + private Optional<ApplicationRevision> applicationRevisionFromSlime(Inspector object) { + if ( ! object.valid()) return Optional.empty(); + String applicationPackageHash = object.field(applicationPackageHashField).asString(); + Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); + return sourceRevision.isPresent() ? Optional.of(ApplicationRevision.from(applicationPackageHash, sourceRevision.get())) + : Optional.of(ApplicationRevision.from(applicationPackageHash)); + } + + private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { + if ( ! object.valid()) return Optional.empty(); + return Optional.of(new SourceRevision(object.field(repositoryField).asString(), + object.field(branchField).asString(), + object.field(commitField).asString())); + } + + private DeploymentJobs deploymentJobsFromSlime(Inspector object) { + Optional<Long> projectId = optionalLong(object.field(projectIdField)); + List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField)); + Optional<String> jiraIssueKey = optionalString(object.field(jiraIssueIdField)); + boolean selfTriggering = object.field(selfTriggeringField).asBool(); + + return new DeploymentJobs(projectId, jobStatusList, jiraIssueKey, selfTriggering); + } + + private Optional<Change> changeFromSlime(Inspector object) { + if ( ! object.valid()) return Optional.empty(); + Inspector versionFieldValue = object.field(versionField); + if (versionFieldValue.valid()) + return Optional.of(new Change.VersionChange(Version.fromString(versionFieldValue.asString()))); + else if (object.field(applicationPackageHashField).valid()) + return Optional.of(Change.ApplicationChange.of(applicationRevisionFromSlime(object).get())); + else + return Optional.of(Change.ApplicationChange.unknown()); + } + + private List<JobStatus> jobStatusListFromSlime(Inspector array) { + List<JobStatus> jobStatusList = new ArrayList<>(); + array.traverse((ArrayTraverser) (int i, Inspector item) -> { + // TODO: This zone has been removed. Remove after Aug 2017 + String jobId = item.field(jobTypeField).asString(); + if ("production-ap-aue-1".equals(jobId)) { + return; + } + jobStatusList.add(jobStatusFromSlime(item)); + }); + return jobStatusList; + } + + private JobStatus jobStatusFromSlime(Inspector object) { + DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromId(object.field(jobTypeField).asString()); + + Optional<JobError> jobError = Optional.empty(); + if (object.field(errorField).valid()) + jobError = Optional.of(JobError.valueOf(object.field(errorField).asString())); + + Inspector versionFieldValue = object.field(versionField); + if (versionFieldValue.valid()) { // TODO: Read legacy JobStatus content: Remove after June 2017 + // Read stored information in old data model + Instant completionTime = Instant.ofEpochMilli(object.field(completionTimeField).asLong()); + Optional<Instant> failingSinceTime = optionalLong(object.field(failingSinceField)).map(Instant::ofEpochMilli); + Optional<Instant> lastTriggeredTime = optionalLong(object.field(lastTriggeredField)).map(Instant::ofEpochMilli); + Version version = new Version(versionFieldValue.asString()); + + // Best-effort conversion to new data model + Optional<JobStatus.JobRun> lastTriggered = lastTriggeredTime.map(at -> new JobStatus.JobRun(version, Optional.empty(), at)); + Optional<JobStatus.JobRun> lastCompleted = Optional.of(new JobStatus.JobRun(version, Optional.empty(), completionTime)); + Optional<JobStatus.JobRun> firstFailing = failingSinceTime.map(at -> new JobStatus.JobRun(version, Optional.empty(), at)); + Optional<JobStatus.JobRun> lastSuccess = Optional.of(new JobStatus.JobRun(version, Optional.empty(), completionTime));; + + return new JobStatus(jobType, jobError, + lastTriggered, lastCompleted, firstFailing, lastSuccess); + } + else { // read current format + return new JobStatus(jobType, jobError, + jobRunFromSlime(object.field(lastTriggeredField)), + jobRunFromSlime(object.field(lastCompletedField)), + jobRunFromSlime(object.field(firstFailingField)), + jobRunFromSlime(object.field(lastSuccessField))); + + } + } + + private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) { + if ( ! object.valid()) return Optional.empty(); + return Optional.of(new JobStatus.JobRun(new Version(object.field(versionField).asString()), + applicationRevisionFromSlime(object.field(revisionField)), + Instant.ofEpochMilli(object.field(atField).asLong()))); + } + + private Optional<Long> optionalLong(Inspector field) { + return field.valid() ? Optional.of(field.asLong()) : Optional.empty(); + } + + private Optional<String> optionalString(Inspector field) { + return SlimeUtils.optionalString(field); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java new file mode 100644 index 00000000000..3fbfdd31808 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java @@ -0,0 +1,74 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.base.Joiner; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Used to store the permanent data of the controller. + * + * @author Stian Kristoffersen + * @author bratseth + */ +public abstract class ControllerDb { + + // --------- Tenants + + public abstract void createTenant(Tenant tenant); + + public abstract void updateTenant(Tenant tenant) throws PersistenceException; + + public abstract void deleteTenant(TenantId tenantId) throws PersistenceException; + + public abstract Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException; + + public abstract List<Tenant> listTenants(); + + // --------- Applications + + // ONLY call this from ApplicationController.store() + public abstract void store(Application application); + + public abstract void deleteApplication(ApplicationId applicationId); + + public abstract Optional<Application> getApplication(ApplicationId applicationId); + + /** Returns all applications */ + public abstract List<Application> listApplications(); + + /** Returns all applications of a tenant */ + public abstract List<Application> listApplications(TenantId tenantId); + + // --------- Rotations + + public abstract Set<RotationId> getRotations(); + + public abstract Set<RotationId> getRotations(ApplicationId applicationId); + + public abstract boolean assignRotation(RotationId rotationId, ApplicationId applicationId); + + public abstract Set<RotationId> deleteRotations(ApplicationId applicationId); + + /** Returns the given elements joined by dot "." */ + protected String path(Identifier... elements) { + return Joiner.on(".").join(elements); + } + + protected String path(String... elements) { + return Joiner.on(".").join(elements); + } + + protected String path(ApplicationId applicationId) { + return applicationId.tenant().value() + "." + applicationId.application().value() + "." + applicationId.instance().value(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java new file mode 100644 index 00000000000..5777636fa24 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -0,0 +1,201 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Curator backed database for storing working state shared between controller servers. + * This maps controller specific operations to general curator operations. + * + * @author bratseth + */ +public class CuratorDb { + + private static final Logger log = Logger.getLogger(CuratorDb.class.getName()); + + private static final Path root = Path.fromString("/controller/v1"); + + private static final Duration defaultLockTimeout = Duration.ofMinutes(5); + + private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); + private final JobQueueSerializer jobQueueSerializer = new JobQueueSerializer(); + + @SuppressWarnings("unused") // This server is used (only) from the curator instance of this over the network */ + //private final ZooKeeperServer zooKeeperServer; + private final Curator curator; + + /** + * All keys, to allow reentrancy. + * This will grow forever, but this should be too slow to be a problem. + */ + private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>(); + + /** Create a curator db which also set up a ZooKeeper server (such that this instance is both client and server) */ + @Inject + public CuratorDb() { + // this.zooKeeperServer = new ZooKeeperServer(createZookeeperServerConfig()); + // this.curator = new Curator("localhost:2281"); + //this.zooKeeperServer = null; + this.curator = new MockCurator(); + } + + private static ZookeeperServerConfig createZookeeperServerConfig() { + ZookeeperServerConfig.Builder b = new ZookeeperServerConfig.Builder(); + b.zooKeeperConfigFile("conf/zookeeper/controller-zookeeper.cfg"); + b.dataDir("var/controller-zookeeper"); + b.clientPort(2281); + b.myidFile("var/controller-zookeeper/myid"); + b.myid(0); + ZookeeperServerConfig.Server.Builder server = new ZookeeperServerConfig.Server.Builder(); + server.id(0); + server.hostname("localhost"); + server.quorumPort(2282); + server.electionPort(2283); + b.server(server); + return new ZookeeperServerConfig(b); + } + + /** Create a curator db which does not set uop a server, using the given Curator instance */ + protected CuratorDb(Curator curator) { + //this.zooKeeperServer = null; + this.curator = curator; + } + + // -------------- Locks -------------------------------------------------- + + public Lock lock(TenantId id, Duration timeout) { + return lock(lockPath(id), timeout); + } + + public Lock lock(ApplicationId id, Duration timeout) { + return lock(lockPath(id), timeout); + } + + /** Create a reentrant lock */ + private Lock lock(Path path, Duration timeout) { + Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), curator)); + lock.acquire(timeout); + return lock; + } + + public Lock lockInactiveJobs() { + return lock(root.append("locks").append("inactiveJobsLock"), defaultLockTimeout); + } + + public Lock lockJobQueues() { + return lock(root.append("locks").append("jobQueuesLock"), defaultLockTimeout); + } + + public Lock lockMaintenanceJob(String jobName) { + // Use a short timeout such that if maintenance jobs are started at about the same time on different nodes + // and the maintenance job takes a long time to complete, only one of the nodes will run the job + // in each maintenance interval + return lock(root.append("locks").append("maintenanceJobLocks").append(jobName), Duration.ofSeconds(1)); + } + + // -------------- Read and write -------------------------------------------------- + + public Version readSystemVersion() { + Optional<byte[]> data = curator.getData(systemVersionPath()); + if (! data.isPresent() || data.get().length == 0) return Vtag.currentVersion; + return Version.fromString(new String(data.get(), StandardCharsets.UTF_8)); + } + + public void writeSystemVersion(Version version) { + NestedTransaction transaction = new NestedTransaction(); + curator.set(systemVersionPath(), version.toString().getBytes(StandardCharsets.UTF_8)); + transaction.commit(); + } + + public Set<String> readInactiveJobs() { + try { + Optional<byte[]> data = curator.getData(inactiveJobsPath()); + if (! data.isPresent() || data.get().length == 0) return new HashSet<>(); // inactive jobs has never been written + return stringSetSerializer.fromJson(data.get()); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Error reading inactive jobs, deleting inactive state"); + writeInactiveJobs(Collections.emptySet()); + return new HashSet<>(); + } + } + + public void writeInactiveJobs(Set<String> inactiveJobs) { + NestedTransaction transaction = new NestedTransaction(); + curator.set(inactiveJobsPath(), stringSetSerializer.toJson(inactiveJobs)); + transaction.commit(); + } + + public Deque<ApplicationId> readJobQueue(DeploymentJobs.JobType jobType) { + try { + Optional<byte[]> data = curator.getData(jobQueuePath(jobType)); + if (! data.isPresent() || data.get().length == 0) return new ArrayDeque<>(); // job queue has never been written + return jobQueueSerializer.fromJson(data.get()); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Error reading job queue, deleting inactive state"); + writeInactiveJobs(Collections.emptySet()); + return new ArrayDeque<>(); + } + } + + public void writeJobQueue(DeploymentJobs.JobType jobType, Deque<ApplicationId> queue) { + NestedTransaction transaction = new NestedTransaction(); + curator.set(jobQueuePath(jobType), jobQueueSerializer.toJson(queue)); + transaction.commit(); + } + + // -------------- Paths -------------------------------------------------- + + private Path systemVersionPath() { + return root.append("systemVersion"); + } + + private Path lockPath(TenantId tenant) { + Path lockPath = root.append("locks") + .append(tenant.id()); + curator.create(lockPath); + return lockPath; + } + + private Path lockPath(ApplicationId application) { + Path lockPath = root.append("locks") + .append(application.tenant().value()) + .append(application.application().value()) + .append(application.instance().value()); + curator.create(lockPath); + return lockPath; + } + + private Path inactiveJobsPath() { + return root.append("inactiveJobs"); + } + + private Path jobQueuePath(DeploymentJobs.JobType jobType) { + return root.append("jobQueues").append(jobType.name()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java new file mode 100644 index 00000000000..5017624f286 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java @@ -0,0 +1,45 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Set; + +/** + * Serialization of a queue of ApplicationIds to/from Json bytes using Slime. + * + * The set is serialized as an array of string. + * + * @author bratseth + */ +public class JobQueueSerializer { + + public byte[] toJson(Deque<ApplicationId> queue) { + try { + Slime slime = new Slime(); + Cursor array = slime.setArray(); + queue.forEach((id -> array.addString(id.serializedForm()))); + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new RuntimeException("Serialization of a job queue failed", e); + } + } + + public Deque<ApplicationId> fromJson(byte[] data) { + Inspector inspector = SlimeUtils.jsonToSlime(data).get(); + Deque<ApplicationId> queue = new ArrayDeque<>(); + inspector.traverse((ArrayTraverser) (index, value) -> queue.addLast(ApplicationId.fromSerializedForm(value.asString()))); + return queue; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java new file mode 100644 index 00000000000..37677a5e393 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java @@ -0,0 +1,132 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.AlreadyExistsException; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A controller db implementation backed by in-memory structures. Useful for testing. + * + * @author Stian Kristoffersen + */ +public class MemoryControllerDb extends ControllerDb { + + private Map<TenantId, Tenant> tenants = new HashMap<>(); + private Map<String, Application> applications = new HashMap<>(); + private Map<RotationId, ApplicationId> rotationAssignments = new HashMap<>(); + + @Override + public void createTenant(Tenant tenant) { + if (tenants.containsKey(tenant.getId())) { + throw new AlreadyExistsException(tenant.getId()); + } + tenants.put(tenant.getId(), tenant); + } + + @Override + public void updateTenant(Tenant tenant) { + if (!tenants.containsKey(tenant.getId())) { + throw new NotExistsException(tenant.getId()); + } + tenants.put(tenant.getId(), tenant); + } + + @Override + public void deleteTenant(TenantId tenantId) { + Object removed = tenants.remove(tenantId); + if (removed == null) + throw new NotExistsException(tenantId); + } + + @Override + public Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException { + Optional<Tenant> tenant = Optional.ofNullable(tenants.get(tenantId)); + if(tenant.isPresent()) { + Tenant t_noquota = tenant.get(); + Tenant t_withquota = new Tenant( + t_noquota.getId(), t_noquota.getUserGroup(), t_noquota.getProperty(), + t_noquota.getAthensDomain(), t_noquota.getPropertyId()); + return Optional.of(t_withquota); + } else { + return tenant; + } + } + + @Override + public List<Tenant> listTenants() { + return new ArrayList<>(tenants.values()); + } + + @Override + public void store(Application application) { + applications.put(path(application.id()), application); + } + + @Override + public void deleteApplication(ApplicationId applicationId) { + applications.remove(path(applicationId)); + } + + @Override + public Optional<Application> getApplication(ApplicationId applicationId) { + return Optional.ofNullable(applications.get(path(applicationId))); + } + + @Override + public List<Application> listApplications() { + return new ArrayList<>(applications.values()); + } + + @Override + public List<Application> listApplications(TenantId tenantId) { + return applications.values().stream() + .filter(a -> a.id().tenant().value().equals(tenantId.id())) + .collect(Collectors.toList()); + } + + @Override + public Set<RotationId> getRotations() { + return rotationAssignments.keySet(); + } + + @Override + public Set<RotationId> getRotations(ApplicationId applicationId) { + return rotationAssignments.entrySet().stream() + .filter(entry -> entry.getValue().equals(applicationId)) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + } + + @Override + public boolean assignRotation(RotationId rotationId, ApplicationId applicationId) { + if (rotationAssignments.containsKey(rotationId)) { + return false; + } else { + rotationAssignments.put(rotationId, applicationId); + return true; + } + } + + @Override + public Set<RotationId> deleteRotations(ApplicationId applicationId) { + Set<RotationId> rotations = getRotations(applicationId); + for (RotationId rotation : rotations) { + rotationAssignments.remove(rotation); + } + return rotations; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java new file mode 100644 index 00000000000..5dc8ca0e545 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.vespa.curator.mock.MockCurator; + +/** + * A curator db backed by a mock curator. + * + * @author bratseth + */ +@SuppressWarnings("unused") // injected +public class MockCuratorDb extends CuratorDb { + + public MockCuratorDb() { + super(new MockCurator()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java new file mode 100644 index 00000000000..b963ecbfab9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java @@ -0,0 +1,19 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +/** + * Exception thrown by persistence layer. + * + * @author mpolden + */ +public class PersistenceException extends Exception { + + public PersistenceException(String message, Throwable cause) { + super(message, cause); + } + + public PersistenceException(Throwable cause) { + super(cause); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java new file mode 100644 index 00000000000..83715e16e8e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * Serialization of a set of strings to/from Json bytes using Slime. + * + * The set is serialized as an array of string. + * + * @author bratseth + */ +public class StringSetSerializer { + + public byte[] toJson(Set<String> stringSet) { + try { + Slime slime = new Slime(); + Cursor array = slime.setArray(); + for (String element : stringSet) + array.addString(element); + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new RuntimeException("Serialization of a string set failed", e); + } + + } + + public Set<String> fromJson(byte[] data) { + Inspector inspector = SlimeUtils.jsonToSlime(data).get(); + Set<String> stringSet = new HashSet<>(); + inspector.traverse((ArrayTraverser) (index, name) -> stringSet.add(name.asString())); + return stringSet; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java new file mode 100644 index 00000000000..87a14660fee --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Persistence layer for the controller. + * + * @author bratseth + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java new file mode 100644 index 00000000000..a9643e21c00 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.yolean.Exceptions; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.FORBIDDEN; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; + +/** + * A HTTP JSON response containing an error code and a message + * + * @author bratseth + */ +public class ErrorResponse extends SlimeJsonResponse { + + public enum errorCodes { + NOT_FOUND, + BAD_REQUEST, + FORBIDDEN, + METHOD_NOT_ALLOWED, + INTERNAL_SERVER_ERROR + } + + public ErrorResponse(int statusCode, String errorType, String message) { + super(statusCode, asSlimeMessage(errorType, message)); + } + + private static Slime asSlimeMessage(String errorType, String message) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("error-code", errorType); + root.setString("message", message); + return slime; + } + + public static ErrorResponse notFoundError(String message) { + return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message); + } + + public static ErrorResponse internalServerError(String message) { + return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message); + } + + public static ErrorResponse badRequest(String message) { + return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message); + } + + public static ErrorResponse forbidden(String message) { + return new ErrorResponse(FORBIDDEN, errorCodes.FORBIDDEN.name(), message); + } + + public static ErrorResponse methodNotAllowed(String message) { + return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message); + } + + public static ErrorResponse from(ConfigServerException e) { + return new ErrorResponse(BAD_REQUEST, e.getErrorCode().name(), Exceptions.toMessageString(e)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java new file mode 100644 index 00000000000..8b2f0e9f09d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author bratseth + */ +public class MessageResponse extends HttpResponse { + + private final Slime slime = new Slime(); + + public MessageResponse(String message) { + super(200); + slime.setObject().setString("message", message); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java new file mode 100644 index 00000000000..c8c027d91c9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java @@ -0,0 +1,109 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A path which is able to match strings containing bracketed placeholders and return the + * values given at the placeholders. + * + * E.g a path /a/1/bar/fuz + * will match /a/{foo}/bar/{b} + * and return foo=1 and b=fuz + * + * Only full path elements may be placeholders, i.e /a{bar} is not interpreted as one. + * + * If the path spec ends with /{*}, it will match urls with any rest path. + * The rest path (not including the trailing slash) will be available as getRest(). + * + * Note that for convenience in common use this has state which is changes as a side effect of each matches + * invocation. It is therefore for single thread use. + * + * @author bratseth + */ +public class Path { + + // This path + private final String pathString; + private final String[] elements; + + // Info about the last match + private final Map<String, String> values = new HashMap<>(); + private String rest = ""; + + public Path(String path) { + this.pathString = path; + this.elements = path.split("/"); + } + + /** + * Returns whether this path matches the given template string. + * If the given template has placeholders, their values (accessible by get) are reset by calling this, + * whether or not the path matches the given template. + * + * This will NOT match empty path elements. + * + * @param pathSpec the path string to match to this + * @return true if the string matches, false otherwise + */ + public boolean matches(String pathSpec) { + values.clear(); + String[] specElements = pathSpec.split("/"); + boolean matchPrefix = false; + if (specElements[specElements.length-1].equals("{*}")) { + matchPrefix = true; + specElements = Arrays.copyOf(specElements, specElements.length-1); + } + + if (matchPrefix) { + if (this.elements.length < specElements.length) return false; + } + else { // match exact + if (this.elements.length != specElements.length) return false; + } + + for (int i = 0; i < specElements.length; i++) { + if (specElements[i].startsWith("{") && specElements[i].endsWith("}")) // placeholder + values.put(specElements[i].substring(1, specElements[i].length()-1), elements[i]); + else if ( ! specElements[i].equals(this.elements[i])) + return false; + } + + if (matchPrefix) { + StringBuilder rest = new StringBuilder(); + for (int i = specElements.length; i < this.elements.length; i++) + rest.append(elements[i]).append("/"); + if ( ! pathString.endsWith("/")) + rest.setLength(rest.length() -1); + this.rest = rest.toString(); + } + + return true; + } + + /** + * Returns the value of the given template variable in the last path matched, or null + * if the previous matches call returned false or if this has not matched anything yet. + */ + public String get(String placeholder) { + return values.get(placeholder); + } + + /** + * Returns the rest of the last matched path. + * This is always the empty string (never null) unless the path spec ends with {*} + */ + public String getRest() { return rest; } + + /** Returns this path as a string */ + public String asString() { return pathString; } + + @Override + public String toString() { + return "path '" + Arrays.stream(elements).collect(Collectors.joining("/")) + "'"; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java new file mode 100644 index 00000000000..550b47d8280 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Returns a response containing an array of links to sub-resources + * + * @author bratseth + */ +public class ResourceResponse extends HttpResponse { + + private final Slime slime = new Slime(); + + public ResourceResponse(HttpRequest request, String ... subResources) { + super(200); + Cursor resourceArray = slime.setObject().setArray("resources"); + for (String subResource : subResources) { + Cursor resourceEntry = resourceArray.addObject(); + resourceEntry.setString("url", new Uri(request.getUri()) + .append(subResource) + .withTrailingSlash() + .toString()); + } + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java new file mode 100644 index 00000000000..9283b1c3018 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java @@ -0,0 +1,96 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.concurrent.Executor; + +/** + * Responds to requests for the root path of the controller by listing the available web service API's. + * + * FAQ: + * - Q: Why do we need this when the container provides a perfectly fine root response listing all handlers by default? + * - A: Because we also have Jersey API's and those are not included in the default response. + * + * @author Oyvind Gronnesby + * @author bratseth + */ +public class RootHandler extends LoggingRequestHandler { + + public RootHandler(Executor executor, AccessLog accessLog) { + super(executor, accessLog); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + final URI requestUri = httpRequest.getUri(); + return new ControllerRootPathResponse(requestUri); + } + + private static class ControllerRootPathResponse extends HttpResponse { + + private final URI uri; + + public ControllerRootPathResponse(URI uri) { + super(200); + this.uri = uri; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + mapper.writeValue(outputStream, buildResponseObject()); + } + + @Override + public String getContentType() { + return "application/json"; + } + + private JsonNode buildResponseObject() { + ObjectNode output = new ObjectNode(JsonNodeFactory.instance); + ArrayNode services = output.putArray("services"); + + jerseyService(services, "provision", "/provision/v1/", "/provision/application.wadl"); + jerseyService(services, "statuspage", "/statuspage/v1/", "/statuspage/application.wadl"); + jerseyService(services, "zone", "/zone/v1/", "/zone/application.wadl"); + jerseyService(services, "zone", "/zone/v2/", "/zone/application.wadl"); + jerseyService(services, "cost", "/cost/v1/", "/cost/application.wadl"); + handlerService(services, "application", "/application/v4/"); + handlerService(services, "deployment", "/deployment/v1/"); + handlerService(services, "screwdriver", "/screwdriver/v1/release/vespa"); + + return output; + } + + private void jerseyService(ArrayNode parent, String name, String url, String wadl) { + ObjectNode service = parent.addObject(); + service.put("name", name); + service.put("url", controllerUri(url)); + service.put("wadl", controllerUri(wadl)); + } + + private void handlerService(ArrayNode parent, String name, String url) { + ObjectNode service = parent.addObject(); + service.put("name", name); + service.put("url", controllerUri(url)); + } + + private String controllerUri(String path) { + return uri.resolve(path).toString(); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java new file mode 100644 index 00000000000..81b07b81efb --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * A generic Json response using Slime for JSON encoding + * + * @author bratseth + */ +public class SlimeJsonResponse extends HttpResponse { + + private final Slime slime; + + public SlimeJsonResponse(Slime slime) { + super(200); + this.slime = slime; + } + + public SlimeJsonResponse(int statusCode, Slime slime) { + super(statusCode); + this.slime = slime; + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java new file mode 100644 index 00000000000..1fc30b7d880 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author bratseth + */ +public class StringResponse extends HttpResponse { + + private final String message; + + public StringResponse(String message) { + super(200); + this.message = message; + } + + @Override + public void render(OutputStream stream) throws IOException { + stream.write(message.getBytes("utf-8")); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java new file mode 100644 index 00000000000..479e7434f9b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * A Uri which provides convenience methods for creating various manipulated copies. + * This is immutable. + * + * @author bratseth + */ +public class Uri { + + /** The URI instance wrapped by this */ + private final URI uri; + + public Uri(URI uri) { + this.uri = uri; + } + + public Uri(String uri) { + try { + this.uri = new URI(uri); + } + catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid URI", e); + } + } + + /** Returns a uri with the given path appended and all parameters removed */ + public Uri append(String pathElement) { + return new Uri(withoutParameters().withTrailingSlash() + pathElement); + } + + public Uri withoutParameters() { + int parameterStart = uri.toString().indexOf("?"); + if (parameterStart < 0) + return new Uri(uri.toString()); + else + return new Uri(uri.toString().substring(0, parameterStart)); + } + + public Uri withPath(String path) { + try { + return new Uri(new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), + uri.getPort(), path, uri.getQuery(), uri.getFragment())); + } + catch (URISyntaxException e) { + throw new IllegalArgumentException("Could not add path '" + path + "' to " + this); + } + } + + public Uri withTrailingSlash() { + if (toString().endsWith("/")) return this; + return new Uri(toString() + "/"); + } + + public URI toURI() { return uri; } + + @Override + public String toString() { return uri.toString(); } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java new file mode 100644 index 00000000000..d701f3d57a0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -0,0 +1,1065 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.google.common.base.Joiner; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.AlreadyExistsException; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.application.v4.ApplicationResource; +import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; +import com.yahoo.vespa.hosted.controller.api.application.v4.TenantResource; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; +import com.yahoo.vespa.hosted.controller.api.integration.cost.CostJsonModelAdapter; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; +import com.yahoo.vespa.hosted.controller.restapi.filter.SetBouncerPassthruHeaderFilter; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.yolean.Exceptions; + +import javax.ws.rs.BadRequestException; +import javax.ws.rs.ForbiddenException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.Principal; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Scanner; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +/** + * This implements the application/v4 API which is used to deploy and manage applications + * on hosted Vespa. + * + * @author bratseth + */ +public class ApplicationApiHandler extends LoggingRequestHandler { + + private final Controller controller; + private final Authorizer authorizer; + + public ApplicationApiHandler(Executor executor, AccessLog accessLog, Controller controller, Authorizer authorizer) { + super(executor, accessLog); + this.controller = controller; + this.authorizer = authorizer; + } + + @Override + public Duration getTimeout() { + return Duration.ofMinutes(20); // deploys may take a long time; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case PUT: return handlePUT(request); + case POST: return handlePOST(request); + case DELETE: return handleDELETE(request); + case OPTIONS: return handleOPTIONS(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (ForbiddenException e) { + return ErrorResponse.forbidden(Exceptions.toMessageString(e)); + } + catch (NotExistsException e) { + return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (ConfigServerException e) { + return ErrorResponse.from(e); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/application/v4/")) return root(request); + if (path.matches("/application/v4/user")) return authenticatedUser(request); + if (path.matches("/application/v4/tenant")) return tenants(request); + if (path.matches("/application/v4/tenant-pipeline")) return tenantPipelines(); + if (path.matches("/application/v4/athensDomain")) return athensDomains(request); + if (path.matches("/application/v4/property")) return properties(request); + if (path.matches("/application/v4/cookiefreshness")) return cookieFreshness(request); + if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), path, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/converge")) return waitForConvergence(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) + return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handlePUT(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/application/v4/user")) return createUser(request); + if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) + return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handlePOST(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/application/v4/tenant/{tenant}/migrateTenantToAthens")) return migrateTenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploy(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/log")) return log(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region")); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handleDELETE(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) + return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handleOPTIONS(HttpRequest request) { + // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother + // spelling out the methods supported at each path, which we should + EmptyJsonResponse response = new EmptyJsonResponse(); + response.headers().put("Allow", "GET,PUT,POST,DELETE,OPTIONS"); + return response; + } + + private HttpResponse root(HttpRequest request) { + return new ResourceResponse(request, + "user", "tenant", "tenant-pipeline", "athensDomain", "property", "cookiefreshness"); + } + + private HttpResponse authenticatedUser(HttpRequest request) { + String userIdString = request.getProperty("userOverride"); + if (userIdString == null) + userIdString = userFrom(request) + .orElseThrow(() -> new ForbiddenException("You must be authenticated or specify userOverride")); + UserId userId = new UserId(userIdString); + + List<Tenant> tenants = controller.tenants().asList(userId); + + Slime slime = new Slime(); + Cursor response = slime.setObject(); + response.setString("user", userId.id()); + Cursor tenantsArray = response.setArray("tenants"); + for (Tenant tenant : tenants) + tenantInTenantsListToSlime(tenant, request.getUri(), tenantsArray.addObject()); + response.setBool("tenantExists", tenants.stream().map(Tenant::getId).anyMatch(id -> id.isTenantFor(userId))); + return new SlimeJsonResponse(slime); + } + + private HttpResponse tenants(HttpRequest request) { + Slime slime = new Slime(); + Cursor response = slime.setArray(); + for (Tenant tenant : controller.tenants().asList()) + tenantInTenantsListToSlime(tenant, request.getUri(), response.addObject()); + return new SlimeJsonResponse(slime); + } + + /** Lists the screwdriver project id for each application */ + private HttpResponse tenantPipelines() { + Slime slime = new Slime(); + Cursor response = slime.setObject(); + Cursor pipelinesArray = response.setArray("tenantPipelines"); + for (Application application : controller.applications().asList()) { + if ( ! application.deploymentJobs().projectId().isPresent()) continue; + + Cursor pipelineObject = pipelinesArray.addObject(); + pipelineObject.setString("screwdriverId", String.valueOf(application.deploymentJobs().projectId().get())); + pipelineObject.setString("tenant", application.id().tenant().value()); + pipelineObject.setString("application", application.id().application().value()); + pipelineObject.setString("instance", application.id().instance().value()); + } + response.setArray("brokenTenantPipelines"); // not used but may need to be present + return new SlimeJsonResponse(slime); + } + + private HttpResponse athensDomains(HttpRequest request) { + Slime slime = new Slime(); + Cursor response = slime.setObject(); + Cursor array = response.setArray("data"); + for (AthensDomain athensDomain : controller.getDomainList(request.getProperty("prefix"))) { + array.addString(athensDomain.id()); + } + return new SlimeJsonResponse(slime); + } + + private HttpResponse properties(HttpRequest request) { + Slime slime = new Slime(); + Cursor response = slime.setObject(); + Cursor array = response.setArray("properties"); + for (Map.Entry<PropertyId, Property> entry : controller.fetchPropertyList().entrySet()) { + Cursor propertyObject = array.addObject(); + propertyObject.setString("propertyid", entry.getKey().id()); + propertyObject.setString("property", entry.getValue().id()); + } + return new SlimeJsonResponse(slime); + } + + private HttpResponse cookieFreshness(HttpRequest request) { + Slime slime = new Slime(); + String passThruHeader = request.getHeader(SetBouncerPassthruHeaderFilter.BOUNCER_PASSTHRU_HEADER_FIELD); + slime.setObject().setBool("shouldRefreshCookie", + ! SetBouncerPassthruHeaderFilter.BOUNCER_PASSTHRU_COOKIE_OK.equals(passThruHeader)); + return new SlimeJsonResponse(slime); + } + + private HttpResponse tenant(String tenantName, HttpRequest request) { + Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(tenantName)); + if ( ! tenant.isPresent()) + return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); + return new SlimeJsonResponse(toSlime(tenant.get(), request, true)); + } + + private HttpResponse applications(String tenantName, HttpRequest request) { + TenantName tenant = TenantName.from(tenantName); + Slime slime = new Slime(); + Cursor array = slime.setArray(); + for (Application application : controller.applications().asList(tenant)) + toSlime(application, array.addObject(), request); + return new SlimeJsonResponse(slime); + } + + private HttpResponse application(String tenantName, String applicationName, Path path, HttpRequest request) { + Slime slime = new Slime(); + Cursor response = slime.setObject(); + + com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"); + Application application = + controller.applications().get(applicationId) + .orElseThrow(() -> new NotExistsException(applicationId + " not found")); + + // Currently deploying change + if (application.deploying().isPresent()) { + Cursor deployingObject = response.setObject("deploying"); + if (application.deploying().get() instanceof Change.VersionChange) + deployingObject.setString("version", ((Change.VersionChange)application.deploying().get()).version().toString()); + else if (((Change.ApplicationChange)application.deploying().get()).revision().isPresent()) + toSlime(((Change.ApplicationChange)application.deploying().get()).revision().get(), deployingObject.setObject("revision")); + } + + // Deployment jobs + Cursor deploymentsArray = response.setArray("deploymentJobs"); + for (JobStatus job : application.deploymentJobs().jobStatus().values()) { + Cursor jobObject = deploymentsArray.addObject(); + jobObject.setString("type", job.type().id()); + jobObject.setBool("success", job.isSuccess()); + + job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered"))); + job.lastCompleted().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastCompleted"))); + job.firstFailing().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("firstFailing"))); + job.lastSuccess().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastSuccess"))); + } + + // Compile version. The version that should be used when building an application + response.setString("compileVersion", application.compileVersion(controller).toFullString()); + + // Rotations + Cursor globalRotationsArray = response.setArray("globalRotations"); + Set<URI> rotations = controller.getRotationUris(applicationId); + Map<String, RotationStatus> rotationHealthStatus = + rotations.isEmpty() ? Collections.emptyMap() : controller.getHealthStatus(rotations.iterator().next().getHost()); + for (URI rotation : rotations) + globalRotationsArray.addString(rotation.toString()); + + // Deployments + Cursor instancesArray = response.setArray("instances"); + for (Deployment deployment : application.deployments().values()) { + Cursor deploymentObject = instancesArray.addObject(); + deploymentObject.setString("environment", deployment.zone().environment().value()); + deploymentObject.setString("region", deployment.zone().region().value()); + deploymentObject.setString("instance", application.id().instance().value()); // pointless + if ( ! rotations.isEmpty()) + setRotationStatus(deployment, rotationHealthStatus, deploymentObject); + deploymentObject.setString("url", withPath(path.asString() + + "/environment/" + deployment.zone().environment().value() + + "/region/" + deployment.zone().region().value() + + "/instance/" + application.id().instance().value(), + request.getUri()).toString()); + } + + // Metrics + try { + MetricsService.ApplicationMetrics metrics = controller.metricsService().getApplicationMetrics(applicationId); + Cursor metricsObject = response.setObject("metrics"); + metricsObject.setDouble("queryServiceQuality", metrics.queryServiceQuality()); + metricsObject.setDouble("writeServiceQuality", metrics.writeServiceQuality()); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed getting Yamas metrics", e); + } + + return new SlimeJsonResponse(slime); + } + + private HttpResponse deployment(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); + Application application = controller.applications().get(id) + .orElseThrow(() -> new NotExistsException(id + " not found")); + + DeploymentId deploymentId = new DeploymentId(application.id(), + new Zone(Environment.from(environment), RegionName.from(region))); + + Deployment deployment = application.deployments().get(deploymentId.zone()); + if (deployment == null) + throw new NotExistsException(application + " is not deployed in " + deploymentId.zone()); + + Optional<InstanceEndpoints> deploymentEndpoints = controller.applications().getDeploymentEndpoints(deploymentId); + + Slime slime = new Slime(); + Cursor response = slime.setObject(); + Cursor serviceUrlArray = response.setArray("serviceUrls"); + if (deploymentEndpoints.isPresent()) { + for (URI uri : deploymentEndpoints.get().getContainerEndpoints()) + serviceUrlArray.addString(uri.toString()); + } + + response.setString("nodes", withPath("/zone/v2/" + environment + "/" + region + "/nodes/v2/node/?&recursive=true&application=" + tenantName + "." + applicationName + "." + instanceName, request.getUri()).toString()); + + Environment env = Environment.from(environment); + RegionName regionName = RegionName.from(region); + URI elkUrl = controller.getElkUri(env, regionName, deploymentId); + if (elkUrl != null) + response.setString("elkUrl", elkUrl.toString()); + + response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); + response.setString("version", deployment.version().toFullString()); + response.setString("revision", deployment.revision().id()); + response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli()); + Optional<Duration> deploymentTimeToLive = controller.zoneRegistry().getDeploymentTimeToLive(Environment.from(environment), RegionName.from(region)); + deploymentTimeToLive.ifPresent(duration -> response.setLong("expiryTimeEpochMs", deployment.at().plus(duration).toEpochMilli())); + + application.deploymentJobs().projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i))); + sourceRevisionToSlime(deployment.revision().source(), response); + + com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, instanceName); + Zone zoneId = new Zone(Environment.from(environment), RegionName.from(region)); + + // Cost + try { + ApplicationCost appCost = controller.getApplicationCost(applicationId, zoneId); + Cursor costObject = response.setObject("cost"); + CostJsonModelAdapter.toSlime(appCost, costObject); + } catch (NotFoundCheckedException nfce) { + log.log(Level.FINE, "Application cost data not found. " + nfce.getMessage()); + } + + // Metrics + try { + MetricsService.DeploymentMetrics metrics = controller.metricsService().getDeploymentMetrics(applicationId, zoneId); + Cursor metricsObject = response.setObject("metrics"); + metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond()); + metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond()); + metricsObject.setDouble("documentCount", metrics.documentCount()); + metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); + metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed getting Yamas metrics", e); + } + + return new SlimeJsonResponse(slime); + } + + private void toSlime(ApplicationRevision revision, Cursor object) { + object.setString("hash", revision.id()); + if (revision.source().isPresent()) + sourceRevisionToSlime(revision.source(), object.setObject("source")); + } + + private void sourceRevisionToSlime(Optional<SourceRevision> revision, Cursor object) { + if ( ! revision.isPresent()) return; + object.setString("gitRepository", revision.get().repository()); + object.setString("gitBranch", revision.get().branch()); + object.setString("gitCommit", revision.get().commit()); + } + + private URI monitoringSystemUri(DeploymentId deploymentId) { + return controller.zoneRegistry().getMonitoringSystemUri(deploymentId.zone().environment(), + deploymentId.zone().region(), + deploymentId.applicationId()); + } + + private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) { + + // Check if request is authorized + Optional<Tenant> existingTenant = controller.tenants().tenant(new TenantId(tenantName)); + if (!existingTenant.isPresent()) + return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist"); + + authorizer.throwIfUnauthorized(existingTenant.get().getId(), request); + + // Decode payload (reason) and construct parameter to the configserver + + Inspector requestData = toSlime(request.getData()).get(); + String reason = mandatory("reason", requestData).asString(); + String agent = authorizer.getUserId(request).toString(); + long timestamp = controller.clock().instant().getEpochSecond(); + EndpointStatus.Status status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; + EndpointStatus endPointStatus = new EndpointStatus(status, reason, agent, timestamp); + + // DeploymentId identifies the zone and application we are dealing with + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + new Zone(Environment.from(environment), RegionName.from(region))); + try { + List<String> rotations = controller.applications().setGlobalRotationStatus(deploymentId, endPointStatus); + return new MessageResponse(String.format("Rotations %s successfully set to %s service", rotations.toString(), inService ? "in" : "out of")); + } catch (IOException e) { + return ErrorResponse.internalServerError("Unable to alter rotation status: " + e.getMessage()); + } + } + + private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { + + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + new Zone(Environment.from(environment), RegionName.from(region))); + + Slime slime = new Slime(); + Cursor c1 = slime.setObject().setArray("globalrotationoverride"); + try { + Map<String, EndpointStatus> rotations = controller.applications().getGlobalRotationStatus(deploymentId); + for (String rotation : rotations.keySet()) { + EndpointStatus currentStatus = rotations.get(rotation); + c1.addString(rotation); + Cursor c2 = c1.addObject(); + c2.setString("status", currentStatus.getStatus().name()); + c2.setString("reason", currentStatus.getReason() == null ? "" : currentStatus.getReason()); + c2.setString("agent", currentStatus.getAgent() == null ? "" : currentStatus.getAgent()); + c2.setLong("timestamp", currentStatus.getEpoch()); + } + } catch (IOException e) { + return ErrorResponse.internalServerError("Unable to get rotation status: " + e.getMessage()); + } + + return new SlimeJsonResponse(slime); + } + + private HttpResponse rotationStatus(String tenantName, String applicationName, String instanceName, String environment, String region) { + + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); + Set<URI> rotations = controller.getRotationUris(applicationId); + if (rotations.isEmpty()) + throw new NotExistsException("global rotation does not exist for '" + environment + "." + region + "'"); + + Slime slime = new Slime(); + Cursor response = slime.setObject(); + + Map<String, RotationStatus> rotationHealthStatus = controller.getHealthStatus(rotations.iterator().next().getHost()); + + for (String rotationEndpoint : rotationHealthStatus.keySet()) { + if (rotationEndpoint.contains(toDns(environment)) && rotationEndpoint.contains(toDns(region))) { + Cursor bcpStatusObject = response.setObject("bcpStatus"); + bcpStatusObject.setString("rotationStatus", rotationHealthStatus.getOrDefault(rotationEndpoint, RotationStatus.UNKNOWN).name()); + } + } + + return new SlimeJsonResponse(slime); + } + + private HttpResponse waitForConvergence(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + return new JacksonJsonResponse(controller.waitForConfigConvergence(new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + new Zone(Environment.from(environment), RegionName.from(region))), + asLong(request.getProperty("timeout"), 1000))); + } + + private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); + ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)), + new com.yahoo.config.provision.ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), + controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)), + request.getUri()); + response.setResponse(applicationView); + return response; + } + + private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { + Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); + ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)), + new com.yahoo.config.provision.ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), + controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)), + request.getUri()); + response.setResponse(result, serviceName, restPath); + return response; + } + + private HttpResponse createUser(HttpRequest request) { + Optional<String> username = userFrom(request); + if ( ! username.isPresent() ) throw new ForbiddenException("Not authenticated."); + + try { + controller.tenants().createUserTenant(username.get()); + return new MessageResponse("Created user '" + username.get() + "'"); + } catch (AlreadyExistsException e) { + // Ok + return new MessageResponse("User '" + username + "' already exists"); + } + } + + private HttpResponse updateTenant(String tenantName, HttpRequest request) { + Optional<Tenant> existingTenant = controller.tenants().tenant(new TenantId(tenantName)); + if ( ! existingTenant.isPresent()) return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");; + + Inspector requestData = toSlime(request.getData()).get(); + + authorizer.throwIfUnauthorized(existingTenant.get().getId(), request); + Tenant updatedTenant; + switch (existingTenant.get().tenantType()) { + case USER: { + throw new BadRequestException("Cannot set property or OpsDB user group for user tenant"); + } + case OPSDB: { + UserGroup userGroup = new UserGroup(mandatory("userGroup", requestData).asString()); + updatedTenant = Tenant.createOpsDbTenant(new TenantId(tenantName), + userGroup, + new Property(mandatory("property", requestData).asString()), + optional("propertyId", requestData).map(PropertyId::new)); + throwIfNotSuperUserOrPartOfOpsDbGroup(userGroup, request); + controller.tenants().updateTenant(updatedTenant, authorizer.getNToken(request)); + break; + } + case ATHENS: { + if (requestData.field("userGroup").valid()) + throw new BadRequestException("Cannot set OpsDB user group to Athens tenant"); + updatedTenant = Tenant.createAthensTenant(new TenantId(tenantName), + new AthensDomain(mandatory("athensDomain", requestData).asString()), + new Property(mandatory("property", requestData).asString()), + optional("propertyId", requestData).map(PropertyId::new)); + controller.tenants().updateTenant(updatedTenant, authorizer.getNToken(request)); + break; + } + default: { + throw new BadRequestException("Unknown tenant type: " + existingTenant.get().tenantType()); + } + } + return new SlimeJsonResponse(toSlime(updatedTenant, request, true)); + } + + private HttpResponse createTenant(String tenantName, HttpRequest request) { + if (new TenantId(tenantName).isUser()) + return ErrorResponse.badRequest("Use User API to create user tenants."); + + Inspector requestData = toSlime(request.getData()).get(); + + Tenant tenant = new Tenant(new TenantId(tenantName), + optional("userGroup", requestData).map(UserGroup::new), + optional("property", requestData).map(Property::new), + optional("athensDomain", requestData).map(AthensDomain::new), + optional("propertyId", requestData).map(PropertyId::new)); + if (tenant.isOpsDbTenant()) + throwIfNotSuperUserOrPartOfOpsDbGroup(new UserGroup(mandatory("userGroup", requestData).asString()), request); + if (tenant.isAthensTenant()) + throwIfNotAthensDomainAdmin(new AthensDomain(mandatory("athensDomain", requestData).asString()), request); + + controller.tenants().addTenant(tenant, authorizer.getNToken(request)); + return new SlimeJsonResponse(toSlime(tenant, request, true)); + } + + private HttpResponse migrateTenant(String tenantName, HttpRequest request) { + TenantId tenantid = new TenantId(tenantName); + Inspector requestData = toSlime(request.getData()).get(); + AthensDomain tenantDomain = new AthensDomain(mandatory("athensDomain", requestData).asString()); + Property property = new Property(mandatory("property", requestData).asString()); + PropertyId propertyId = new PropertyId(mandatory("propertyId", requestData).asString()); + + authorizer.throwIfUnauthorized(tenantid, request); + throwIfNotAthensDomainAdmin(tenantDomain, request); + NToken nToken = authorizer.getNToken(request) + .orElseThrow(() -> + new BadRequestException("The NToken for a domain admin is required to migrate tenant to Athens")); + Tenant tenant = controller.tenants().migrateTenantToAthens(tenantid, tenantDomain, propertyId, property, nToken); + return new SlimeJsonResponse(toSlime(tenant, request, true)); + } + + private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { + authorizer.throwIfUnauthorized(new TenantId(tenantName), request); + Application application; + try { + application = controller.applications().createApplication(com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"), authorizer.getNToken(request)); + } + catch (ZmsException e) { // TODO: Push conversion down + if (e.getCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) + throw new ForbiddenException("Not authorized to create application", e); + else + throw e; + } + + Slime slime = new Slime(); + toSlime(application, slime.setObject(), request); + return new SlimeJsonResponse(slime); + } + + /** Trigger deployment of the last built application package, on a given version */ + private HttpResponse deploy(String tenantName, String applicationName, HttpRequest request) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + try (Lock lock = controller.applications().lock(id)) { + Application application = controller.applications().require(id); + if (application.deploying().isPresent()) + throw new IllegalArgumentException("Can not start a deployment of " + application + " at this time: " + + application.deploying() + " is in progress"); + + Version version = decideDeployVersion(request); + if ( ! systemHasVersion(version)) + throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + + "Version is not active in this system. " + + "Active versions: " + controller.versionStatus().versions()); + + // Since we manually triggered it we don't want this to be self-triggering for the time being + controller.applications().store(application.with(application.deploymentJobs().asSelfTriggering(false)), lock); + + controller.applications().deploymentTrigger().triggerChange(application.id(), new Change.VersionChange(version)); + return new MessageResponse("Triggered deployment of " + application + " on version " + version); + } + } + + private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + new Zone(Environment.from(environment), RegionName.from(region))); + // TODO: Propagate all filters + if (request.getProperty("hostname") != null) + controller.applications().restartHost(deploymentId, new Hostname(request.getProperty("hostname"))); + else + controller.applications().restart(deploymentId); + + // TODO: Change to return JSON + return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName, + ApplicationResource.API_PATH, applicationName, + EnvironmentResource.API_PATH, environment, + "region", region, + "instance", instanceName)); + } + + /** + * This returns and deletes recent error logs from this deployment, which is used by tenant deployment jobs to verify that + * the application is working. It is called for all production zones, also those in which the application is not present, + * and possibly before it is present, so failures are normal and expected. + */ + private HttpResponse log(String tenantName, String applicationName, String instanceName, String environment, String region) { + try { + DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), + new Zone(Environment.from(environment), RegionName.from(region))); + return new JacksonJsonResponse(controller.grabLog(deploymentId)); + } + catch (RuntimeException e) { + Slime slime = new Slime(); + slime.setObject(); + return new SlimeJsonResponse(slime); + } + } + + private HttpResponse deploy(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); + Zone zone = new Zone(Environment.from(environment), RegionName.from(region)); + + Map<String, byte[]> dataParts = new MultipartParser().parse(request); + if ( ! dataParts.containsKey("deployOptions")) + return ErrorResponse.badRequest("Missing required form part 'deployOptions'"); + if ( ! dataParts.containsKey("applicationZip")) + return ErrorResponse.badRequest("Missing required form part 'applicationZip'"); + + Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); + + DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.athens(), controller.zoneRegistry()); + Tenant tenant = controller.tenants().tenant(new TenantId(tenantName)).orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); + Principal principal = authorizer.getPrincipal(request); + if (principal instanceof AthensPrincipal) { + deployAuthorizer.throwIfUnauthorizedForDeploy(principal, + Environment.from(environment), + tenant, + applicationId); + } else { // In case of host-based principal + UserId userId = new UserId(principal.getName()); + deployAuthorizer.throwIfUnauthorizedForDeploy( + Environment.from(environment), + userId, + tenant, + applicationId, + optional("screwdriverBuildJob", deployOptions).map(ScrewdriverId::new)); + } + + + // TODO: get rid of the json object + DeployOptions deployOptionsJsonClass = new DeployOptions(screwdriverBuildJobFromSlime(deployOptions.field("screwdriverBuildJob")), + optional("vespaVersion", deployOptions).map(Version::new), + deployOptions.field("ignoreValidationErrors").asBool(), + deployOptions.field("deployCurrentVersion").asBool()); + ActivateResult result = controller.applications().deployApplication(applicationId, + zone, + new ApplicationPackage(dataParts.get("applicationZip")), + deployOptionsJsonClass); + return new SlimeJsonResponse(toSlime(result, dataParts.get("applicationZip").length)); + } + + private HttpResponse deleteTenant(String tenantName, HttpRequest request) { + Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(tenantName)); + if ( ! tenant.isPresent()) return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); // NOTE: The Jersey implementation would silently ignore this + + authorizer.throwIfUnauthorized(new TenantId(tenantName), request); + controller.tenants().deleteTenant(new TenantId(tenantName), authorizer.getNToken(request)); + + // TODO: Change to a message response saying the tenant was deleted + return new SlimeJsonResponse(toSlime(tenant.get(), request, false)); + } + + private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { + authorizer.throwIfUnauthorized(new TenantId(tenantName), request); + + com.yahoo.config.provision.ApplicationId id = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"); + Application deleted = controller.applications().deleteApplication(id, authorizer.getNToken(request)); + if (deleted == null) + return ErrorResponse.notFoundError("Could not delete application '" + id + "': Application not found"); + return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead + } + + private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region) { + Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); + + Zone zone = new Zone(Environment.from(environment), RegionName.from(region)); + Deployment deployment = application.deployments().get(zone); + if (deployment == null) + return ErrorResponse.notFoundError("Could not deactivate: " + application + " is not deployed in " + zone); + + controller.applications().deactivate(application, deployment, false); + + // TODO: Change to return JSON + return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, + ApplicationResource.API_PATH, applicationName, + EnvironmentResource.API_PATH, environment, + "region", region, + "instance", instanceName)); + } + + /** + * Promote application Chef environments. To be used by component jobs only + */ + private HttpResponse promoteApplication(String tenantName, String applicationName) { + try{ + ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); + String sourceEnvironment = chefEnvironment.systemChefEnvironment(); + String targetEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); + controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); + return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); + } catch (Exception e) { + log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s)", tenantName, applicationName), e); + return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); + } + } + + /** + * Promote application Chef environments for jobs that deploy applications + */ + private HttpResponse promoteApplicationDeployment(String tenantName, String applicationName, String environmentName, String regionName) { + try { + ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); + String sourceEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); + String targetEnvironment = chefEnvironment.applicationTargetEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName), Environment.from(environmentName), RegionName.from(regionName)); + controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); + return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); + } catch (Exception e) { + log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s %s.%s)", tenantName, applicationName, environmentName, regionName), e); + return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); + } + } + + private Optional<String> userFrom(HttpRequest request) { + return authorizer.getPrincipalIfAny(request).map(Principal::getName); + } + + private void toSlime(Tenant tenant, Cursor object, HttpRequest request, boolean listApplications) { + object.setString("type", tenant.tenantType().name()); + tenant.getAthensDomain().ifPresent(a -> object.setString("athensDomain", a.id())); + tenant.getProperty().ifPresent(p -> object.setString("property", p.id())); + tenant.getPropertyId().ifPresent(p -> object.setString("propertyId", p.toString())); + tenant.getUserGroup().ifPresent(g -> object.setString("userGroup", g.id())); + Cursor applicationArray = object.setArray("applications"); + if (listApplications) { // This cludge is needed because we call this after deleting the tenant. As this call makes another tenant lookup it will fail. TODO is to support lookup on tenant + for (Application application : controller.applications().asList(TenantName.from(tenant.getId().id()))) { + if (application.id().instance().isDefault()) // TODO: Skip non-default applications until supported properly + toSlime(application, applicationArray.addObject(), request); + } + } + } + + // A tenant has different content when in a list ... antipattern, but not solvable before application/v5 + private void tenantInTenantsListToSlime(Tenant tenant, URI requestURI, Cursor object) { + object.setString("tenant", tenant.getId().id()); + Cursor metaData = object.setObject("metaData"); + metaData.setString("type", tenant.tenantType().name()); + tenant.getAthensDomain().ifPresent(a -> metaData.setString("athensDomain", a.id())); + tenant.getProperty().ifPresent(p -> metaData.setString("property", p.id())); + tenant.getUserGroup().ifPresent(g -> metaData.setString("userGroup", g.id())); + object.setString("url", withPath("/application/v4/tenant/" + tenant.getId().id(), requestURI).toString()); + } + + /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ + private URI withPath(String newPath, URI uri) { + try { + return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); + } + catch (URISyntaxException e) { + throw new RuntimeException("Will not happen", e); + } + } + + private void setRotationStatus(Deployment deployment, Map<String, RotationStatus> healthStatus, Cursor object) { + if ( ! deployment.zone().environment().equals(Environment.prod)) return; + + Cursor bcpStatusObject = object.setObject("bcpStatus"); + bcpStatusObject.setString("rotationStatus", findRotationStatus(deployment, healthStatus).name()); + } + + private RotationStatus findRotationStatus(Deployment deployment, Map<String, RotationStatus> healthStatus) { + for (String endpoint : healthStatus.keySet()) { + if (endpoint.contains(toDns(deployment.zone().environment().value())) && + endpoint.contains(toDns(deployment.zone().region().value()))) { + return healthStatus.getOrDefault(endpoint, RotationStatus.UNKNOWN); + } + } + + return RotationStatus.UNKNOWN; + } + + private String toDns(String id) { + return id.replace('_', '-'); + } + + private long asLong(String valueOrNull, long defaultWhenNull) { + if (valueOrNull == null) return defaultWhenNull; + try { + return Long.parseLong(valueOrNull); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Expected an integer but got '" + valueOrNull + "'"); + } + } + + private void toSlime(JobStatus.JobRun jobRun, Cursor object) { + object.setString("version", jobRun.version().toFullString()); + jobRun.revision().ifPresent(revision -> toSlime(revision, object.setObject("revision"))); + object.setLong("at", jobRun.at().toEpochMilli()); + } + + private Slime toSlime(InputStream jsonStream) { + try { + byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); + return SlimeUtils.jsonToSlime(jsonBytes); + } catch (IOException e) { + throw new RuntimeException(); + } + } + + private void throwIfNotSuperUserOrPartOfOpsDbGroup(UserGroup userGroup, HttpRequest request) { + UserId userId = authorizer.getUserId(request); + if (!authorizer.isSuperUser(request) && !authorizer.isGroupMember(userId, userGroup) ) { + throw new ForbiddenException(String.format("User '%s' is not super user or part of the OpsDB user group '%s'", + userId.id(), userGroup.id())); + } + } + + private void throwIfNotAthensDomainAdmin(AthensDomain tenantDomain, HttpRequest request) { + UserId userId = authorizer.getUserId(request); + if ( ! authorizer.isAthensDomainAdmin(userId, tenantDomain)) { + throw new ForbiddenException( + String.format("The user '%s' is not admin in Athens domain '%s'", userId.id(), tenantDomain.id())); + } + } + + private Inspector mandatory(String key, Inspector object) { + if ( ! object.field(key).valid()) + throw new IllegalArgumentException("'" + key + "' is missing"); + return object.field(key); + } + + private Optional<String> optional(String key, Inspector object) { + return SlimeUtils.optionalString(object.field(key)); + } + + private static String path(Object... elements) { + return Joiner.on("/").join(elements); + } + + private Slime toSlime(Tenant tenant, HttpRequest request, boolean listApplications) { + Slime slime = new Slime(); + toSlime(tenant, slime.setObject(), request, listApplications); + return slime; + } + + private void toSlime(Application application, Cursor object, HttpRequest request) { + object.setString("application", application.id().application().value()); + object.setString("instance", application.id().instance().value()); + object.setString("url", withPath("/application/v4/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value(), request.getUri()).toString()); + } + + private Slime toSlime(ActivateResult result, long applicationZipSizeBytes) { + Slime slime = new Slime(); + Cursor object = slime.setObject(); + object.setString("revisionId", result.getRevisionId().id()); + object.setLong("applicationZipSize", applicationZipSizeBytes); + Cursor logArray = object.setArray("prepareMessages"); + if (result.getPrepareResponse().log != null) { + for (Log logMessage : result.getPrepareResponse().log) { + Cursor logObject = logArray.addObject(); + logObject.setLong("time", logMessage.time); + logObject.setString("level", logMessage.level); + logObject.setString("message", logMessage.message); + } + } + + Cursor changeObject = object.setObject("configChangeActions"); + + Cursor restartActionsArray = changeObject.setArray("restart"); + for (RestartAction restartAction : result.getPrepareResponse().configChangeActions.restartActions) { + Cursor restartActionObject = restartActionsArray.addObject(); + restartActionObject.setString("clusterName", restartAction.clusterName); + restartActionObject.setString("clusterType", restartAction.clusterType); + restartActionObject.setString("serviceType", restartAction.serviceType); + serviceInfosToSlime(restartAction.services, restartActionObject.setArray("services")); + stringsToSlime(restartAction.messages, restartActionObject.setArray("messages")); + } + + Cursor refeedActionsArray = changeObject.setArray("refeed"); + for (RefeedAction refeedAction : result.getPrepareResponse().configChangeActions.refeedActions) { + Cursor refeedActionObject = refeedActionsArray.addObject(); + refeedActionObject.setString("name", refeedAction.name); + refeedActionObject.setBool("allowed", refeedAction.allowed); + refeedActionObject.setString("documentType", refeedAction.documentType); + refeedActionObject.setString("clusterName", refeedAction.clusterName); + serviceInfosToSlime(refeedAction.services, refeedActionObject.setArray("services")); + stringsToSlime(refeedAction.messages, refeedActionObject.setArray("messages")); + } + return slime; + } + + private void serviceInfosToSlime(List<ServiceInfo> serviceInfoList, Cursor array) { + for (ServiceInfo serviceInfo : serviceInfoList) { + Cursor serviceInfoObject = array.addObject(); + serviceInfoObject.setString("serviceName", serviceInfo.serviceName); + serviceInfoObject.setString("serviceType", serviceInfo.serviceType); + serviceInfoObject.setString("configId", serviceInfo.configId); + serviceInfoObject.setString("hostName", serviceInfo.hostName); + } + } + + private void stringsToSlime(List<String> strings, Cursor array) { + for (String string : strings) + array.addString(string); + } + + // TODO: get rid of the json object + private Optional<ScrewdriverBuildJob> screwdriverBuildJobFromSlime(Inspector object) { + if ( ! object.valid() ) return Optional.empty(); + Optional<ScrewdriverId> screwdriverId = optional("screwdriverId", object).map(ScrewdriverId::new); + return Optional.of(new ScrewdriverBuildJob(screwdriverId.orElse(null), + gitRevisionFromSlime(object.field("gitRevision")))); + } + + // TODO: get rid of the json object + private GitRevision gitRevisionFromSlime(Inspector object) { + return new GitRevision(optional("repository", object).map(GitRepository::new).orElse(null), + optional("branch", object).map(GitBranch::new).orElse(null), + optional("commit", object).map(GitCommit::new).orElse(null)); + } + + private String readToString(InputStream stream) { + Scanner scanner = new Scanner(stream).useDelimiter("\\A"); + if ( ! scanner.hasNext()) return null; + return scanner.next(); + } + + private boolean systemHasVersion(Version version) { + return controller.versionStatus().versions().stream().anyMatch(v -> v.versionNumber().equals(version)); + } + + private Version decideDeployVersion(HttpRequest request) { + String requestVersion = readToString(request.getData()); + if (requestVersion != null) + return new Version(requestVersion); + else + return controller.systemVersion(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java new file mode 100644 index 00000000000..7c32e48e218 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; + +/** + * Represents Chef environments for applications/deployments. Used for promotion of Chef environments + * + * @author mortent + */ +public class ApplicationChefEnvironment { + + private final String systemChefEnvironment; + private final String systemSuffix; + + public ApplicationChefEnvironment(SystemName system) { + if (system == SystemName.main) { + systemChefEnvironment = "hosted-verified-prod"; + systemSuffix = ""; + } else { + systemChefEnvironment = "hosted-infra-cd"; + systemSuffix = "-cd"; + } + } + + public String systemChefEnvironment() { + return systemChefEnvironment; + } + + public String applicationSourceEnvironment(TenantName tenantName, ApplicationName applicationName) { + // placeholder and component already used in legacy chef promotion + return String.format("hosted-instance%s_%s_%s_placeholder_component_default", systemSuffix, tenantName, applicationName); + } + + public String applicationTargetEnvironment(TenantName tenantName, ApplicationName applicationName, Environment environment, RegionName regionName) { + return String.format("hosted-instance%s_%s_%s_%s_%s_default", systemSuffix, tenantName, applicationName, regionName, environment); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java new file mode 100644 index 00000000000..8dff39779b9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java @@ -0,0 +1,164 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.Environment; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.common.ContextAttributes; +import com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter; +import com.yahoo.vespa.hosted.controller.restapi.filter.UnauthenticatedUserPrincipal; + +import javax.ws.rs.ForbiddenException; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.SecurityContext; +import java.security.Principal; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; + + +/** + * @author Stian Kristoffersen + * @author Tony Vaagenes + * @author bjorncs + */ +// TODO: Make this an interface +public class Authorizer { + + private static final Logger log = Logger.getLogger(Authorizer.class.getName()); + + // Must be kept in sync with bouncer filter configuration. + private static final String VESPA_HOSTED_ADMIN_ROLE = "10707.A"; + + private static final Set<UserId> SCREWDRIVER_USERS = ImmutableSet.of(new UserId("screwdrv"), + new UserId("screwdriver"), + new UserId("sdrvtest"), + new UserId("screwdriver-test")); + + private final Controller controller; + private final ZmsClientFactory zmsClientFactory; + private final EntityService entityService; + private final Athens athens; + + public Authorizer(Controller controller, EntityService entityService) { + this.controller = controller; + this.zmsClientFactory = controller.athens().zmsClientFactory(); + this.entityService = entityService; + this.athens = controller.athens(); + } + + public void throwIfUnauthorized(TenantId tenantId, HttpRequest request) throws ForbiddenException { + if (isReadOnlyMethod(request.getMethod().name())) return; + if (isSuperUser(request)) return; + + Optional<Tenant> tenant = controller.tenants().tenant(tenantId); + if ( ! tenant.isPresent()) return; + + UserId userId = getUserId(request); + if (isTenantAdmin(userId, tenant.get())) return; + + throw loggedForbiddenException("User " + userId + " does not have write access to tenant " + tenantId); + } + + public UserId getUserId(HttpRequest request) { + String name = getPrincipal(request).getName(); + if (name == null) + throw loggedForbiddenException("Not authorized: User name is null"); + return new UserId(name); + } + + /** Returns the principal or throws forbidden */ // TODO: Avoid REST exceptions + public Principal getPrincipal(HttpRequest request) { + return getPrincipalIfAny(request).orElseThrow(() -> Authorizer.loggedForbiddenException("User is not authenticated")); + } + + /** Returns the principal if there is any */ + public Optional<Principal> getPrincipalIfAny(HttpRequest request) { + return securityContextOf(request).map(SecurityContext::getUserPrincipal); + } + + public Optional<NToken> getNToken(HttpRequest request) { + String nTokenHeader = (String)request.getJDiscRequest().context().get(NTokenRequestFilter.NTOKEN_HEADER); + return Optional.ofNullable(nTokenHeader).map(athens::nTokenFrom); + } + + public boolean isSuperUser(HttpRequest request) { + // TODO Check membership of admin role in Vespa's Athens domain + return isMemberOfVespaBouncerGroup(request) || isScrewdriverPrincipal(athens, getPrincipal(request)); + } + + public static boolean isScrewdriverPrincipal(Athens athens, Principal principal) { + if (principal instanceof UnauthenticatedUserPrincipal) // Host-based authentication + return SCREWDRIVER_USERS.contains(new UserId(principal.getName())); + else if (principal instanceof AthensPrincipal) + return ((AthensPrincipal)principal).getDomain().equals(athens.screwdriverDomain()); + else + return false; + } + + private static ForbiddenException loggedForbiddenException(String message, Object... args) { + String formattedMessage = String.format(message, args); + log.info(formattedMessage); + return new ForbiddenException(formattedMessage); + } + + private boolean isTenantAdmin(UserId userId, Tenant tenant) { + switch (tenant.tenantType()) { + case ATHENS: + return isAthensTenantAdmin(userId, tenant.getAthensDomain().get()); + case OPSDB: + return isGroupMember(userId, tenant.getUserGroup().get()); + case USER: + return isUserTenantOwner(tenant.getId(), userId); + } + throw new IllegalArgumentException("Unknown tenant type: " + tenant.tenantType()); + } + + private boolean isAthensTenantAdmin(UserId userId, AthensDomain tenantDomain) { + return zmsClientFactory.createClientWithServicePrincipal() + .hasTenantAdminAccess(athens.principalFrom(userId), tenantDomain); + } + + public boolean isAthensDomainAdmin(UserId userId, AthensDomain tenantDomain) { + return zmsClientFactory.createClientWithServicePrincipal() + .isDomainAdmin(athens.principalFrom(userId), tenantDomain); + } + + public boolean isGroupMember(UserId userId, UserGroup userGroup) { + return entityService.isGroupMember(userId, userGroup); + } + + private static boolean isUserTenantOwner(TenantId tenantId, UserId userId) { + return tenantId.equals(userId.toTenantId()); + } + + public static boolean environmentRequiresAuthorization(Environment environment) { + return environment != Environment.dev && environment != Environment.perf; + } + + private static boolean isReadOnlyMethod(String method) { + return method.equals(HttpMethod.GET) || method.equals(HttpMethod.HEAD) || method.equals(HttpMethod.OPTIONS); + } + + private boolean isMemberOfVespaBouncerGroup(HttpRequest request) { + Optional<SecurityContext> securityContext = securityContextOf(request); + if ( ! securityContext.isPresent() ) throw Authorizer.loggedForbiddenException("User is not authenticated"); + return securityContext.get().isUserInRole(Authorizer.VESPA_HOSTED_ADMIN_ROLE); + } + + protected Optional<SecurityContext> securityContextOf(HttpRequest request) { + return Optional.ofNullable((SecurityContext)request.getJDiscRequest().context().get(ContextAttributes.SECURITY_CONTEXT_ATTRIBUTE)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java new file mode 100644 index 00000000000..5c7cdfdae0a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java @@ -0,0 +1,117 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.restapi.filter.UnauthenticatedUserPrincipal; + +import javax.ws.rs.ForbiddenException; +import java.security.Principal; +import java.util.Optional; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.controller.restapi.application.Authorizer.environmentRequiresAuthorization; +import static com.yahoo.vespa.hosted.controller.restapi.application.Authorizer.isScrewdriverPrincipal; + +/** + * @author bjorncs + * @author gjoranv + */ +public class DeployAuthorizer { + + private static final Logger log = Logger.getLogger(DeployAuthorizer.class.getName()); + + private final Athens athens; + private final ZoneRegistry zoneRegistry; + + public DeployAuthorizer(Athens athens, ZoneRegistry zoneRegistry) { + this.athens = athens; + this.zoneRegistry = zoneRegistry; + } + + public void throwIfUnauthorizedForDeploy(Principal principal, + Environment environment, + Tenant tenant, + ApplicationId applicationId) { + if (athensCredentialsRequired(environment, tenant, applicationId, principal)) + checkAthensCredentials(principal, tenant, applicationId); + } + + // TODO: inline when deployment via ssh is removed + private boolean athensCredentialsRequired(Environment environment, Tenant tenant, ApplicationId applicationId, Principal principal) { + if (!environmentRequiresAuthorization(environment)) return false; + + if (! isScrewdriverPrincipal(athens, principal)) + throw loggedForbiddenException( + "Principal '%s' is not a screwdriver principal, and does not have deploy access to application '%s'", + principal.getName(), applicationId.toShortString()); + + return tenant.isAthensTenant(); + } + + + // TODO: inline when deployment via ssh is removed + private void checkAthensCredentials(Principal principal, Tenant tenant, ApplicationId applicationId) { + AthensDomain domain = tenant.getAthensDomain().get(); + if (! (principal instanceof AthensPrincipal)) + throw loggedForbiddenException("Principal '%s' is not authenticated.", principal.getName()); + + AthensPrincipal athensPrincipal = (AthensPrincipal)principal; + if ( ! hasDeployAccessToAthensApplication(athensPrincipal, domain, applicationId)) + throw loggedForbiddenException( + "Screwdriver principal '%1$s' does not have deploy access to '%2$s'. " + + "Either the application has not been created at " + zoneRegistry.getDashboardUri() + " or " + + "'%1$s' is not added to the application's deployer role in Athens domain '%3$s'.", + athensPrincipal, applicationId, tenant.getAthensDomain().get()); + } + + private static ForbiddenException loggedForbiddenException(String message, Object... args) { + String formattedMessage = String.format(message, args); + log.info(formattedMessage); + return new ForbiddenException(formattedMessage); + } + + /** + * @deprecated Only usable for ssh. Use the method that takes Principal instead of UserId and screwdriverId. + */ + @Deprecated + public void throwIfUnauthorizedForDeploy(Environment environment, + UserId userId, + Tenant tenant, + ApplicationId applicationId, + Optional<ScrewdriverId> optionalScrewdriverId) { + + Principal principal = new UnauthenticatedUserPrincipal(userId.id()); + + if (athensCredentialsRequired(environment, tenant, applicationId, principal)) { + ScrewdriverId screwdriverId = optionalScrewdriverId.orElseThrow( + () -> loggedForbiddenException("Screwdriver id must be provided when deploying from Screwdriver.")); + principal = athens.principalFrom(screwdriverId); + checkAthensCredentials(principal, tenant, applicationId); + } + } + + private boolean hasDeployAccessToAthensApplication(AthensPrincipal principal, AthensDomain domain, ApplicationId applicationId) { + try { + return athens.zmsClientFactory().createClientWithServicePrincipal() + .hasApplicationAccess( + principal, + ApplicationAction.deploy, + domain, + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value())); + } catch (ZmsException e) { + throw loggedForbiddenException( + "Failed to authorize deployment through Athens. If this problem persists, " + + "please create ticket at yo/vespa-support. (" + e.getMessage() + ")"); + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java new file mode 100644 index 00000000000..3e8d4182c42 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java @@ -0,0 +1,25 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author bratseth + */ +public class EmptyJsonResponse extends HttpResponse { + + public EmptyJsonResponse() { + super(200); + } + + @Override + public void render(OutputStream stream) throws IOException { } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java new file mode 100644 index 00000000000..cfd6feccf01 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author bratseth + */ +public class JacksonJsonResponse extends HttpResponse { + + private final JsonNode node; + + public JacksonJsonResponse(JsonNode node) { + super(200); + this.node = node; + } + + @Override + public void render(OutputStream stream) throws IOException { + new ObjectMapper().writeValue(stream, node); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java new file mode 100644 index 00000000000..75f4ff68f1e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java @@ -0,0 +1,72 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.container.jdisc.HttpRequest; +import org.apache.commons.fileupload.MultipartStream; +import org.apache.commons.fileupload.ParameterParser; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Provides reading a multipart/form-data request type into a map of bytes for each part, + * indexed by the parts (form field) name. + * + * @author bratseth + */ +public class MultipartParser { + + /** + * Parses the given multi-part request and returns all the parts indexed by their name. + * + * @throws IllegalArgumentException if this request is not a well-formed request with Content-Type multipart/form-data + */ + public Map<String, byte[]> parse(HttpRequest request) { + try { + ParameterParser parameterParser = new ParameterParser(); + Map<String, String> contentType = parameterParser.parse(request.getHeader("Content-Type"), ';'); + if ( ! contentType.containsKey("multipart/form-data")) + throw new IllegalArgumentException("Expected a multipart message, but got Content-Type: " + + request.getHeader("Content-Type")); + String boundary = contentType.get("boundary"); + if (boundary == null) + throw new IllegalArgumentException("Missing boundary property in Content-Type header"); + MultipartStream multipartStream = new MultipartStream(request.getData(), boundary.getBytes(), + 1000 * 1000, + null); + boolean nextPart = multipartStream.skipPreamble(); + Map<String, byte[]> parts = new HashMap<>(); + while (nextPart) { + String[] headers = multipartStream.readHeaders().split("\r\n"); + String contentDispositionContent = findContentDispositionHeader(headers); + if (contentDispositionContent == null) + throw new IllegalArgumentException("Missing Content-Disposition header in a multipart body part"); + Map<String, String> contentDisposition = parameterParser.parse(contentDispositionContent, ';'); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + multipartStream.readBodyData(output); + parts.put(contentDisposition.get("name"), output.toByteArray()); + nextPart = multipartStream.readBoundary(); + } + return parts; + } + catch(MultipartStream.MalformedStreamException e) { + throw new IllegalArgumentException("Malformed multipart/form-data request", e); + } + catch(IOException e) { + throw new IllegalArgumentException("IO error reading multipart request " + request.getUri(), e); + } + } + + private String findContentDispositionHeader(String[] headers) { + String contentDisposition = "Content-Disposition:"; + for (String header : headers) { + if (header.length() < contentDisposition.length()) continue; + if ( ! header.substring(0, contentDisposition.length()).equalsIgnoreCase(contentDisposition)) continue; + return header.substring(contentDisposition.length() + 1); + } + return null; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java new file mode 100644 index 00000000000..6a448e475c5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -0,0 +1,191 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.restapi.Uri; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.ClusterView; +import com.yahoo.vespa.serviceview.bindings.ServiceView; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A response containing a service view for an application deployment. + * This does not define the API response but merely proxies the API response provided by Vespa, with URLs + * rewritten to include zone and application information allow proxying through the controller + * + * @author Steinar Knutsen + * @author bratseth + */ +class ServiceApiResponse extends HttpResponse { + + private final Zone zone; + private final ApplicationId application; + private final List<URI> configServerURIs; + private final Slime slime; + private final Uri requestUri; + + // Only set for one of the setResponse calls + private String serviceName = null; + private String restPath = null; + + public ServiceApiResponse(Zone zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { + super(200); + this.zone = zone; + this.application = application; + this.configServerURIs = configServerURIs; + this.slime = new Slime(); + this.requestUri = new Uri(requestUri).withoutParameters(); + } + + public void setResponse(ApplicationView applicationView) { + Cursor clustersArray = slime.setObject().setArray("clusters"); + for (ClusterView clusterView : applicationView.clusters) { + Cursor clusterObject = clustersArray.addObject(); + clusterObject.setString("name", clusterView.name); + clusterObject.setString("type", clusterView.type); + setNullableString("url", rewriteIfUrl(clusterView.url, requestUri), clusterObject); + Cursor servicesArray = clusterObject.setArray("services"); + for (ServiceView serviceView : clusterView.services) { + Cursor serviceObject = servicesArray.addObject(); + setNullableString("url", rewriteIfUrl(serviceView.url, requestUri), serviceObject); + serviceObject.setString("serviceType", serviceView.serviceType); + serviceObject.setString("serviceName", serviceView.serviceName); + serviceObject.setString("configId", serviceView.configId); + serviceObject.setString("host", serviceView.host); + } + } + } + + public void setResponse(Map<?,?> responseData, String serviceName, String restPath) { + this.serviceName = serviceName; + this.restPath = restPath; + mapToSlime(responseData, slime.setObject()); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + @SuppressWarnings("unchecked") + private void mapToSlime(Map<?,?> data, Cursor object) { + for (Map.Entry<String, Object> entry : ((Map<String, Object>)data).entrySet()) + fieldToSlime(entry.getKey(), entry.getValue(), object); + } + + private void fieldToSlime(String key, Object value, Cursor object) { + if (value instanceof String) { + if (key.equals("url") || key.equals("link")) + value = rewriteIfUrl((String)value, generateLocalLinkPrefix(serviceName, restPath)); + setNullableString(key, (String)value, object); + } + else if (value instanceof Integer) { + object.setLong(key, (int)value); + } + else if (value instanceof Long) { + object.setLong(key, (long)value); + } + else if (value instanceof Float) { + object.setDouble(key, (double)value); + } + else if (value instanceof Double) { + object.setDouble(key, (double)value); + } + else if (value instanceof List) { + listToSlime((List)value, object.setArray(key)); + } + else if (value instanceof Map) { + mapToSlime((Map<?,?>)value, object.setObject(key)); + } + } + + private void listToSlime(List<?> list, Cursor array) { + for (Object entry : list) + entryToSlime(entry, array); + } + + private void entryToSlime(Object entry, Cursor array) { + if (entry instanceof String) + addNullableString(rewriteIfUrl((String)entry, generateLocalLinkPrefix(serviceName, restPath)), array); + else if (entry instanceof Integer) + array.addLong((long)entry); + else if (entry instanceof Long) + array.addLong((long)entry); + else if (entry instanceof Float) + array.addDouble((double)entry); + else if (entry instanceof Double) + array.addDouble((double)entry); + else if (entry instanceof List) + listToSlime((List)entry, array.addArray()); + else if (entry instanceof Map) + mapToSlime((Map)entry, array.addObject()); + } + + private String rewriteIfUrl(String urlOrAnyString, Uri requestUri) { + if (urlOrAnyString == null) return null; + + String hostPattern = "(" + + String.join( + "|", configServerURIs.stream() + .map(URI::toString) + .map(s -> s.substring(0, s.length() -1)) + .map(Pattern::quote) + .toArray(String[]::new)) + + ")"; + + String remoteServicePath = "/serviceview/" + + "v1/tenant/" + application.tenant().value() + + "/application/" + application.application().value() + + "/environment/" + zone.environment().value() + + "/region/" + zone.region().value() + + "/instance/" + application.instance() + + "/service/"; + + Pattern remoteServiceResourcePattern = Pattern.compile("^(" + hostPattern + Pattern.quote(remoteServicePath) + ")"); + Matcher matcher = remoteServiceResourcePattern.matcher(urlOrAnyString); + + if (matcher.find()) { + String proxiedPath = urlOrAnyString.substring(matcher.group().length()); + return requestUri.append(proxiedPath).toString(); + } else { + return urlOrAnyString; // not a service url + } + } + + private Uri generateLocalLinkPrefix(String identifier, String restPath) { + String proxiedPath = identifier + "/" + restPath; + + if (this.requestUri.toString().endsWith(proxiedPath)) { + return new Uri(this.requestUri.toString().substring(0, this.requestUri.toString().length() - proxiedPath.length())); + } else { + throw new IllegalStateException("Expected the resource path '" + this.requestUri + "' to end with '" + proxiedPath + "'"); + } + } + + private void setNullableString(String key, String valueOrNull, Cursor receivingObject) { + if (valueOrNull == null) + receivingObject.setNix(key); + else + receivingObject.setString(key, valueOrNull); + } + + private void addNullableString(String valueOrNull, Cursor receivingArray) { + if (valueOrNull == null) + receivingArray.addNix(); + else + receivingArray.addString(valueOrNull); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java new file mode 100644 index 00000000000..e02a31440ce --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -0,0 +1,84 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.controller; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; +import com.yahoo.yolean.Exceptions; + +import java.util.concurrent.Executor; +import java.util.logging.Level; + +/** + * This implements the controller/v1 API which provides operators with information about, + * and control over the Controller. + * + * @author bratseth + */ +public class ControllerApiHandler extends LoggingRequestHandler { + + private final ControllerMaintenance maintenance; + + public ControllerApiHandler(Executor executor, AccessLog accessLog, ControllerMaintenance maintenance) { + super(executor, accessLog); + this.maintenance = maintenance; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case POST: return handlePOST(request); + case DELETE: return handleDELETE(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/controller/v1/")) return root(request); + if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl()); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handlePOST(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/controller/v1/maintenance/inactive/{jobName}")) + return setActive(path.get("jobName"), false); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handleDELETE(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/controller/v1/maintenance/inactive/{jobName}")) + return setActive(path.get("jobName"), true); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse root(HttpRequest request) { + return new ResourceResponse(request, "maintenance"); + } + + private HttpResponse setActive(String jobName, boolean active) { + if ( ! maintenance.jobControl().jobs().contains(jobName)) + return ErrorResponse.notFoundError("No job named '" + jobName + "'"); + maintenance.jobControl().setActive(jobName, active); + return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java new file mode 100644 index 00000000000..e7d1b3e0ed8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java @@ -0,0 +1,46 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.controller; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * A response containing maintenance job status + * + * @author bratseth + */ +public class JobsResponse extends HttpResponse { + + private final JobControl jobControl; + + public JobsResponse(JobControl jobControl) { + super(200); + this.jobControl = jobControl; + } + + @Override + public void render(OutputStream stream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor jobArray = root.setArray("jobs"); + for (String jobName : jobControl.jobs()) + jobArray.addObject().setString("name", jobName); + + Cursor inactiveArray = root.setArray("inactive"); + for (String jobName : jobControl.inactiveJobs()) + inactiveArray.addString(jobName); + + new JsonFormat(true).encode(stream, slime); + } + + @Override + public String getContentType() { return "application/json"; } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java new file mode 100644 index 00000000000..affd679f2c2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -0,0 +1,122 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.Uri; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.yolean.Exceptions; + +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; + +/** + * This implements the deployment/v1 API which provides information about the status of Vespa platform and + * application deployments. + * + * @author bratseth + */ +public class DeploymentApiHandler extends LoggingRequestHandler { + + private final Controller controller; + + public DeploymentApiHandler(Executor executor, AccessLog accessLog, Controller controller) { + super(executor, accessLog); + this.controller = controller; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return handleGET(request); + case OPTIONS: return handleOPTIONS(); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } + catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse handleGET(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/deployment/v1/")) return root(request); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse handleOPTIONS() { + // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother + // spelling out the methods supported at each path, which we should + EmptyJsonResponse response = new EmptyJsonResponse(); + response.headers().put("Allow", "GET,OPTIONS"); + return response; + } + + private HttpResponse root(HttpRequest request) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor platformArray = root.setArray("versions"); + for (VespaVersion version : controller.versionStatus().versions()) { + Cursor versionObject = platformArray.addObject(); + versionObject.setString("version", version.versionNumber().toString()); + versionObject.setString("confidence", version.confidence().name()); + versionObject.setString("commit", version.releaseCommit()); + versionObject.setLong("date", version.releasedAt().toEpochMilli()); + versionObject.setBool("controllerVersion", version.isSelfVersion()); + versionObject.setBool("systemVersion", version.isCurrentSystemVersion()); + + Cursor configServerArray = versionObject.setArray("configServers"); + for (String configServerHostnames : version.configServerHostnames()) { + Cursor configServerObject = configServerArray.addObject(); + configServerObject.setString("hostname", configServerHostnames); + } + + Cursor failingArray = versionObject.setArray("failingApplications"); + for (ApplicationId id : version.statistics().failing()) { + Optional<Application> application = controller.applications().get(id); + if ( ! application.isPresent()) continue; // deleted just now + + Instant failingSince = application.get().deploymentJobs().failingSince(); + if (failingSince == null) continue; // started working just now + + Cursor applicationObject = failingArray.addObject(); + toSlime(id, applicationObject, request); + applicationObject.setLong("failingSince", failingSince.toEpochMilli()); + } + + Cursor productionArray = versionObject.setArray("productionApplications"); + for (ApplicationId id : version.statistics().production()) + toSlime(id, productionArray.addObject(), request); + } + return new SlimeJsonResponse(slime); + } + + private void toSlime(ApplicationId id, Cursor object, HttpRequest request) { + object.setString("tenant", id.tenant().value()); + object.setString("application", id.application().value()); + object.setString("instance", id.instance().value()); + object.setString("url", new Uri(request.getUri()).withPath("/application/v4" + + "/tenant/" + id.tenant().value() + + "/application/" + id.application().value()) + .toString()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java new file mode 100644 index 00000000000..aea59c16cd5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +import static java.util.concurrent.TimeUnit.DAYS; + +/** + * @author gv + */ +public interface AccessControlHeaders { + + String CORS_PREFLIGHT_REQUEST_CACHE_TTL = Long.toString(DAYS.toSeconds(7)); + + String ALLOW_ORIGIN_HEADER = "Access-Control-Allow-Origin"; + + Map<String, String> ACCESS_CONTROL_HEADERS = ImmutableMap.of( + "Access-Control-Max-Age", CORS_PREFLIGHT_REQUEST_CACHE_TTL, + "Access-Control-Allow-Headers", "Origin,Content-Type,Accept,Yahoo-Principal-Auth", + "Access-Control-Allow-Methods", "OPTIONS,GET,PUT,DELETE,POST", + "Access-Control-Allow-Credentials", "true" + ); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java new file mode 100644 index 00000000000..8dace5d56dc --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java @@ -0,0 +1,68 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.google.inject.Inject; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.HttpResponse; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig; +import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Before; + +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS; +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER; + +/** + * <p> + * This filter makes sure we respond as quickly as possible to CORS pre-flight requests + * which browsers transmit before the Hosted Vespa dashboard code is allowed to send a "real" request. + * </p> + * <p> + * An "Access-Control-Max-Age" header is added so that the browser will cache the result of this pre-flight request, + * further improving the responsiveness of the Hosted Vespa dashboard application. + * </p> + * <p> + * Runs after all standard security request filters, but before BouncerFilter, as the browser does not send + * credentials with pre-flight requests. + * </p> + * + * @author andreer + * @author gv + */ +@After({"InputValidationFilter","RemoteIPFilter", "DoNotTrackRequestFilter", "CookieDataRequestFilter"}) +@Before("BouncerFilter") +public class AccessControlRequestFilter implements SecurityRequestFilter { + private final Set<String> allowedUrls; + + @Inject + public AccessControlRequestFilter(HttpAccessControlConfig config) { + allowedUrls = Collections.unmodifiableSet(config.allowedUrls().stream().collect(Collectors.toSet())); + } + + @Override + public void filter(DiscFilterRequest discFilterRequest, ResponseHandler responseHandler) { + String origin = discFilterRequest.getHeader("Origin"); + + if (!discFilterRequest.getMethod().equals(OPTIONS.name())) + return; + + HttpResponse response = HttpResponse.newInstance(Response.Status.OK); + + if (allowedUrls.contains(origin)) + response.headers().add(ALLOW_ORIGIN_HEADER, origin); + + ACCESS_CONTROL_HEADERS.forEach( + (name, value) -> response.headers().add(name, value)); + + ContentChannel cc = responseHandler.handleResponse(response); + cc.close(null); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java new file mode 100644 index 00000000000..c2ad31cd925 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.jdisc.AbstractResource; +import com.yahoo.jdisc.http.filter.DiscFilterResponse; +import com.yahoo.jdisc.http.filter.RequestView; +import com.yahoo.jdisc.http.filter.SecurityResponseFilter; +import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig; + +import java.util.List; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS; +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER; + +/** + * @author gv + * @author Tony Vaagenes + */ +public class AccessControlResponseFilter extends AbstractResource implements SecurityResponseFilter { + + private final List<String> allowedUrls; + + public AccessControlResponseFilter(HttpAccessControlConfig config) { + allowedUrls = config.allowedUrls(); + } + + @Override + public void filter(DiscFilterResponse response, RequestView request) { + Optional<String> requestOrigin = request.getFirstHeader("Origin"); + + requestOrigin.ifPresent( + origin -> allowedUrls.stream() + .filter(allowedUrl -> matchesRequestOrigin(origin, allowedUrl)) + .findAny() + .ifPresent(allowedOrigin -> setHeaderUnlessExists(response, ALLOW_ORIGIN_HEADER, allowedOrigin)) + ); + ACCESS_CONTROL_HEADERS.forEach((name, value) -> setHeaderUnlessExists(response, name, value)); + } + + private boolean matchesRequestOrigin(String requestOrigin, String allowedUrl) { + return allowedUrl.equals("*") || requestOrigin.startsWith(allowedUrl); + } + + /** + * This is to avoid duplicating headers already set by the {@link AccessControlRequestFilter}. + * Currently (March 2016), this filter is invoked for OPTIONS requests to jdisc request handlers, + * even if the request filter has been invoked first. For jersey based APIs, this filter is NOT + * invoked in these cases. + */ + private void setHeaderUnlessExists(DiscFilterResponse response, String name, String value) { + if (response.getHeader(name) == null) + response.setHeader(name, value); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java new file mode 100644 index 00000000000..7beb3f755ad --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; + +/** + * @author Stian Kristoffersen + */ +public class DummyFilter implements SecurityRequestFilter { + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + /* Do nothing - a bug in JDisc prevents empty request chains */ + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java new file mode 100644 index 00000000000..0138d3ae65c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.google.inject.Inject; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.yolean.chain.After; + +/** + * @author bjorncs + */ +@After("BouncerFilter") +public class NTokenRequestFilter implements SecurityRequestFilter { + + public static final String NTOKEN_HEADER = "com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter.ntoken"; + + private final Athens athens; + + @Inject + public NTokenRequestFilter(Athens athens) { + this.athens = athens; + } + + @Override + public void filter(DiscFilterRequest request, ResponseHandler responseHandler) { + String nToken = request.getHeader(athens.principalTokenHeader()); + if (nToken != null) { + request.setAttribute(NTOKEN_HEADER, nToken); + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java new file mode 100644 index 00000000000..7ea98528a88 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.yolean.chain.After; + +/** + * @author Stian Kristoffersen + */ +@After("BouncerFilter") +public class SetBouncerPassthruHeaderFilter implements SecurityRequestFilter { + + public static final String BOUNCER_PASSTHRU_ATTRIBUTE = "bouncer.bypassthru"; + public static final String BOUNCER_PASSTHRU_COOKIE_OK = "1"; + public static final String BOUNCER_PASSTHRU_HEADER_FIELD = "com.yahoo.hosted.vespa.bouncer.passthru"; + + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + Object statusProperty = request.getAttribute(BOUNCER_PASSTHRU_ATTRIBUTE); + String status = Integer.toString((int)statusProperty); + + request.addHeader(BOUNCER_PASSTHRU_HEADER_FIELD, status); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java new file mode 100644 index 00000000000..a88e881ce9d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import java.security.Principal; +import java.util.Objects; + +/** + * A principal for an unauthenticated user (typically from a trusted host). + * This principal should only be used in combination with machine authentication! + * + * @author bjorncs + */ +public class UnauthenticatedUserPrincipal implements Principal { + private final String username; + + public UnauthenticatedUserPrincipal(String username) { + this.username = username; + } + + @Override + public String getName() { + return username; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UnauthenticatedUserPrincipal that = (UnauthenticatedUserPrincipal) o; + return Objects.equals(username, that.username); + } + + @Override + public int hashCode() { + return Objects.hash(username); + } + + @Override + public String toString() { + return "UnauthenticatedUserPrincipal{" + + "username='" + username + '\'' + + '}'; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java new file mode 100644 index 00000000000..46df4d7a603 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.hosted.controller.api.nonpublic.HeaderFields; +import com.yahoo.yolean.chain.Before; + +/** + * Allows hosts using host-based authentication to set user ID. + * + * @author Tony Vaagenes + */ +@Before("CreateSecurityContextFilter") +public class UserIdRequestFilter implements SecurityRequestFilter { + + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + String userName = request.getHeader(HeaderFields.USER_ID_HEADER_FIELD); + request.setUserPrincipal(new UnauthenticatedUserPrincipal(userName)); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java new file mode 100644 index 00000000000..850130ca970 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext; + +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.hosted.controller.common.ContextAttributes; +import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Provides; + +import javax.ws.rs.core.SecurityContext; +import java.security.Principal; + +/** + * Exposes the security information from the disc filter request + * by storing a security context in the request context. + * + * @author Tony Vaagenes + */ +@After("BouncerFilter") +@Provides("SecurityContext") +public class CreateSecurityContextFilter implements SecurityRequestFilter { + + @Override + public void filter(DiscFilterRequest request, ResponseHandler handler) { + request.setAttribute(ContextAttributes.SECURITY_CONTEXT_ATTRIBUTE, + new SecurityContext() { + @Override + public Principal getUserPrincipal() { + return request.getUserPrincipal(); + } + + @Override + public boolean isUserInRole(String role) { + return request.isUserInRole(role); + } + + @Override + public boolean isSecure() { + return request.isSecure(); + } + + @Override + public String getAuthenticationScheme() { + throw new UnsupportedOperationException(); + } + }); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java new file mode 100644 index 00000000000..17c86e89362 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java @@ -0,0 +1,31 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext; + +import com.yahoo.vespa.hosted.controller.common.ContextAttributes; + +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.PreMatching; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.ext.Provider; +import java.io.IOException; + +/** + * Get the security context from the underlying Servlet request, and expose it to + * Jersey resources. + * + * @author Tony Vaagenes + */ +@PreMatching +@Provider +public class PropagateSecurityContextFilter implements ContainerRequestFilter { + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + SecurityContext securityContext = + (SecurityContext) requestContext.getProperty(ContextAttributes.SECURITY_CONTEXT_ATTRIBUTE); + + if (securityContext != null) { + requestContext.setSecurityContext(securityContext); + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java new file mode 100644 index 00000000000..0b98599dbb0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Jersey requires that the package is exported to be able to instantiate the filter. + * + * @author Tony Vaagenes + */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java new file mode 100644 index 00000000000..a623e880c4c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -0,0 +1,168 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.screwdriver; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This implements a callback API from Screwdriver which lets deployment jobs notify the controller + * on completion. + * + * @author bratseth + */ +public class ScrewdriverApiHandler extends LoggingRequestHandler { + + private final static Logger log = Logger.getLogger(ScrewdriverApiHandler.class.getName()); + + private final Controller controller; + // TODO: Remember to distinguish between PR jobs and component ones, by adding reports to the right jobs? + + public ScrewdriverApiHandler(Executor executor, AccessLog accessLog, Controller controller) { + super(executor, accessLog); + this.controller = controller; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + Method method = request.getMethod(); + String path = request.getUri().getPath(); + switch (method) { + case GET: switch (path) { + case "/screwdriver/v1/release/vespa": return vespaVersion(); + case "/screwdriver/v1/jobsToRun": return buildJobResponse(controller.applications().deploymentTrigger().buildSystem().jobs()); + default: return ErrorResponse.notFoundError(String.format( "No '%s' handler at '%s'", method, path)); + } + case POST: switch (path) { + case "/screwdriver/v1/jobreport": return handleJobReportPost(request); + default: return ErrorResponse.notFoundError(String.format( "No '%s' handler at '%s'", method, path)); + } + case DELETE: switch (path) { + case "/screwdriver/v1/jobsToRun": return buildJobResponse(controller.applications().deploymentTrigger().buildSystem().takeJobsToRun()); + default: return ErrorResponse.notFoundError(String.format( "No '%s' handler at '%s'", method, path)); + } + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + } + } catch (IllegalArgumentException|IllegalStateException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse vespaVersion() { + VespaVersion version = controller.versionStatus().version(controller.systemVersion()); + if (version == null) + return ErrorResponse.notFoundError("Information about the current system version is not available at this time"); + + Slime slime = new Slime(); + Cursor cursor = slime.setObject(); + cursor.setString("version", version.versionNumber().toString()); + cursor.setString("sha", version.releaseCommit()); + cursor.setLong("date", version.releasedAt().toEpochMilli()); + return new SlimeJsonResponse(slime); + + } + + private HttpResponse buildJobResponse(List<BuildJob> buildJobs) { + Slime slime = new Slime(); + Cursor buildJobArray = slime.setArray(); + for (BuildJob buildJob : buildJobs) { + Cursor buildJobObject = buildJobArray.addObject(); + buildJobObject.setLong("projectId", buildJob.projectId()); + buildJobObject.setString("jobName", buildJob.jobName()); + } + return new SlimeJsonResponse(slime); + } + + /** + * Parse a JSON blob of the form: + * { + * "tenant" : String + * "application" : String + * "instance" : String + * "jobName" : String + * "projectId" : long + * "buildNumber" : long + * "success" : boolean + * "selfTriggering": boolean + * "gitChanges" : boolean + * "vespaVersion" : String + * } + * and notify the controller of the report. + * + * @param request The JSON blob. + * @return 200 + */ + private HttpResponse handleJobReportPost(HttpRequest request) { + // TODO: buildNumber is unused now -- remove, or use. + // TODO: selfTriggering is unused now -- remove, or use. + // TODO: gitChanges is unused now -- remove, or use. + // Note: gitChanges is probably only useful for the component step, since it check the gir repo directly; + // for other jobs, the last component's git commit is what matters. + // TODO: ApplicationId (tenant, application, instance) is unused now -- remove, or use. + + controller.applications().notifyJobCompletion(toJobReport(toSlime(request.getData()).get())); + + return new StringResponse("ok"); + } + + private Slime toSlime(InputStream jsonStream) { + try { + byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); + return SlimeUtils.jsonToSlime(jsonBytes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private JobReport toJobReport(Inspector report) { + Optional<JobError> jobError = Optional.empty(); + if (report.field("jobError").valid()) { + jobError = Optional.of(JobError.valueOf(report.field("jobError").asString())); + } else if (report.field("success").valid()) { // TODO: Remove after May 2017 + jobError = JobError.from(report.field("success").asBool()); + } + return new JobReport( + ApplicationId.from( + report.field("tenant").asString(), + report.field("application").asString(), + report.field("instance").asString()), + JobType.fromId(report.field("jobName").asString()), + report.field("projectId").asLong(), + report.field("buildNumber").asLong(), + jobError, + report.field("selfTriggering").asBool(), + report.field("gitChanges").asBool() + ); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java new file mode 100644 index 00000000000..fbd1a74c12c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -0,0 +1,62 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; + +import java.util.List; + +/** + * Statistics about deployments on a platform version. This is immutable. + * + * @author bratseth + */ +public class DeploymentStatistics { + + private final Version version; + private final ImmutableList<ApplicationId> failing; + private final ImmutableList<ApplicationId> production; + + private DeploymentStatistics(Version version, + List<ApplicationId> failingApplications, List<ApplicationId> production) { + this.version = version; + this.failing = ImmutableList.copyOf(failingApplications); + this.production = ImmutableList.copyOf(production); + } + + /** Returns a statistics instance with the values as 0 */ + public static DeploymentStatistics empty(Version version) { + return new DeploymentStatistics(version, ImmutableList.of(), ImmutableList.of()); + } + + /** Returns the version these statistics are for */ + public Version version() { return version; } + + /** + * Returns the applications which have at least one job (of any type) which fails on this version, + * excluding errors known to not be caused by this version + */ + public List<ApplicationId> failing() { return failing; } + + /** Returns the applications which have this version in production in at least one zone */ + public List<ApplicationId> production() { return production; } + + /** Returns a version of this with the given failing application added */ + public DeploymentStatistics withFailing(ApplicationId application) { + return new DeploymentStatistics(version, add(application, failing), production); + } + + /** Returns a version of this with the given production application added */ + public DeploymentStatistics withProduction(ApplicationId application) { + return new DeploymentStatistics(version, failing, add(application, production)); + } + + private ImmutableList<ApplicationId> add(ApplicationId application, ImmutableList<ApplicationId> list) { + ImmutableList.Builder<ApplicationId> b = new ImmutableList.Builder<>(); + b.addAll(list); + b.add(application); + return b.build(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java new file mode 100644 index 00000000000..bef96014e79 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -0,0 +1,182 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import com.google.common.collect.ImmutableList; +import com.yahoo.collections.ListMap; +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; +import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.JobStatus; + +import java.net.URI; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Information about the current platform versions in use. + * The versions in use are the set of all versions running in current applications, versions + * of config servers in all zones, and the version of this controller itself. + * + * This is immutable. + * + * @author bratseth + */ +public class VersionStatus { + + private static final Logger log = Logger.getLogger(VersionStatus.class.getName()); + + private static final String VESPA_REPO = "vespa-yahoo"; + private static final String VESPA_REPO_OWNER = "vespa"; + + private final ImmutableList<VespaVersion> versions; + + /** Create a version status. DO NOT USE: Public for testing only */ + public VersionStatus(List<VespaVersion> versions) { + this.versions = ImmutableList.copyOf(versions); + } + + /** + * Returns the current Vespa version of the system controlled by this, + * or empty if we have not currently determined what the system version is in this status. + */ + public Optional<VespaVersion> systemVersion() { + return versions().stream().filter(VespaVersion::isCurrentSystemVersion).findAny(); + } + + /** + * Lists all currently active Vespa versions, with deployment statistics, + * sorted from lowest to highest version number. + * The returned list is immutable. + * Calling this is free, but the returned status is slightly out of date. + */ + public List<VespaVersion> versions() { return versions; } + + /** Returns the given version, or null if it is not present */ + public VespaVersion version(Version version) { + return versions.stream().filter(v -> v.versionNumber().equals(version)).findFirst().orElse(null); + } + + /** Create the empty version status */ + public static VersionStatus empty() { return new VersionStatus(ImmutableList.of()); } + + /** Create a full, updated version status. This is expensive and should be done infrequently */ + public static VersionStatus compute(Controller controller) { + return compute(controller, Vtag.currentVersion); + } + + /** Compute version status using the given current version. This is useful for testing. */ + public static VersionStatus compute(Controller controller, Version currentVersion) { + ListMap<Version, String> configServerVersions = findConfigServerVersions(controller); + + Set<Version> infrastructureVersions = new HashSet<>(); + infrastructureVersions.add(currentVersion); + infrastructureVersions.addAll(configServerVersions.keySet()); + + // The system version is the oldest infrastructure version + Version systemVersion = infrastructureVersions.stream().sorted().findFirst().get(); + + Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, + controller.applications().asList()); + List<VespaVersion> versions = new ArrayList<>(); + + for (DeploymentStatistics statistics : deploymentStatistics) { + if (statistics.version().isEmpty()) continue; + + try { + VespaVersion vespaVersion = createVersion(statistics, + statistics.version().equals(systemVersion), + configServerVersions.getList(statistics.version()), + controller); + versions.add(vespaVersion); + } catch (IllegalArgumentException e) { + log.log(Level.WARNING, "Unable to create VespaVersion for version " + + statistics.version().toFullString(), e); + } + } + Collections.sort(versions); + + return new VersionStatus(versions); + } + + private static ListMap<Version, String> findConfigServerVersions(Controller controller) { + List<URI> configServers = controller.zoneRegistry().zones().stream() + .flatMap(zone -> controller.getConfigServerUris(zone.environment(), zone.region()).stream()) + .collect(Collectors.toList()); + + ListMap<Version, String> versions = new ListMap<>(); + for (URI configServer : configServers) + versions.put(controller.applications().configserverClient().version(configServer), configServer.getHost()); + return versions; + } + + private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions, + List<Application> applications) { + Map<Version, DeploymentStatistics> versionMap = new HashMap<>(); + + for (Version infrastructureVersion : infrastructureVersions) + versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion)); + + for (Application application : applications) { + DeploymentJobs jobs = application.deploymentJobs(); + + // Note that each version deployed on this application exists + for (Deployment deployment : application.deployments().values()) + versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty); + + // List versions which have failing jobs, and versions which are in production + // TODO: Don't count applications which started failing on an application change, not a version change + + // Failing versions + Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream() + .filter(jobStatus -> jobStatus.lastCompleted().isPresent()) + .filter(jobStatus -> jobStatus.jobError().isPresent()) + .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity) + .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version())); + for (Version v : failingJobsByVersion.keySet()) { + versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withFailing(application.id())); + } + + // Succeeding versions + Map<Version, List<JobStatus>> succeedingJobsByVersions = jobs.jobStatus().values().stream() + .filter(jobStatus -> jobStatus.lastSuccess().isPresent()) + .filter(jobStatus -> jobStatus.type().isProduction()) + .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastSuccess().get().version())); + for (Version v : succeedingJobsByVersions.keySet()) { + versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withProduction(application.id())); + } + } + return versionMap.values(); + } + + private static DeploymentStatistics emptyIfMissing(Version version, DeploymentStatistics statistics) { + return statistics == null ? DeploymentStatistics.empty(version) : statistics; + } + + private static VespaVersion createVersion(DeploymentStatistics statistics, + boolean isSystemVersion, + Collection<String> configServerHostnames, + Controller controller) { + GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); + return new VespaVersion(statistics, + gitSha.sha, Instant.ofEpochMilli(gitSha.commit.author.date.getTime()), + isSystemVersion, + configServerHostnames, + controller); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java new file mode 100644 index 00000000000..ce5533bd0bc --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -0,0 +1,139 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; +import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; + +import java.time.Instant; +import java.util.Collection; +import java.util.Set; + +/** + * Information about a particular Vespa version. + * VespaVersions are identified by their version number and ordered by increasing version numbers. + * + * This is immutable. + * + * @author bratseth + */ +public class VespaVersion implements Comparable<VespaVersion> { + + private final String releaseCommit; + private final Instant releasedAt; + private final boolean isCurrentSystemVersion; + private final DeploymentStatistics statistics; + private final Confidence confidence; + private final ImmutableSet<String> configServerHostnames; + + public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt, + boolean isCurrentSystemVersion, Collection<String> configServerHostnames, + Controller controller) { + this.statistics = statistics; + this.releaseCommit = releaseCommit; + this.releasedAt = releasedAt; + this.isCurrentSystemVersion = isCurrentSystemVersion; + this.configServerHostnames = ImmutableSet.copyOf(configServerHostnames); + this.confidence = deduceConfidenceFrom(statistics, controller, releasedAt); + } + + private static Confidence deduceConfidenceFrom(DeploymentStatistics statistics, + Controller controller, + Instant releasedAt) { + // 'production on this': All deployment jobs upgrading to this version have completed without failure + ApplicationList productionOnThis = ApplicationList.from(statistics.production(), controller.applications()) + .notUpgradingTo(statistics.version()) + .notFailing(); + ApplicationList failingOnThis = ApplicationList.from(statistics.failing(), controller.applications()); + ApplicationList all = ApplicationList.from(controller.applications().asList()) + .hasDeployment() + .notPullRequest(); + + // 'broken' if any Canary fails + if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty()) + return Confidence.broken; + + // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all + int brokenByThisVersion = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt).size(); + if (brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1) + return Confidence.broken; + + // 'low' unless all canary applications are upgraded + if (productionOnThis.with(UpgradePolicy.canary).size() < all.with(UpgradePolicy.canary).size()) + return Confidence.low; + + // 'high' if 90% of all default upgrade applications upgraded + if (productionOnThis.with(UpgradePolicy.defaultPolicy).size() >= + all.with(UpgradePolicy.defaultPolicy).size() * 0.9) + return Confidence.high; + + return Confidence.normal; + } + + /** Returns the version number of this Vespa version */ + public Version versionNumber() { return statistics.version(); } + + /** Returns the sha of the release tag commit for this version in git */ + public String releaseCommit() { return releaseCommit; } + + /** Returns the time of the release commit */ + public Instant releasedAt() { return releasedAt; } + + /** Statistics about deployment of this version */ + public DeploymentStatistics statistics() { return statistics; } + + /** Returns whether this is the version currently running on this controller */ + public boolean isSelfVersion() { return versionNumber().equals(Vtag.currentVersion); } + + /** + * Returns whether this is the current version of the infrastructure of the system + * (i.e the lowest version across this controller and all config servers in all zones). + * A goal of the controller is to eventually (limited by safety and upgrade capacity) drive + * all applications to this version. + * + * Note that the self version may be higher than the current system version if + * all config servers are not yet upgraded to the version of this controller. + */ + public boolean isCurrentSystemVersion() { return isCurrentSystemVersion; } + + /** Returns the host names of the config servers (across all zones) which are currently of this version */ + public Set<String> configServerHostnames() { return configServerHostnames; } + + /** Returns the confidence we have in this versions suitability for production */ + public Confidence confidence() { return confidence; } + + @Override + public int compareTo(VespaVersion other) { + return this.versionNumber().compareTo(other.versionNumber()); + } + + @Override + public int hashCode() { return versionNumber().hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof VespaVersion)) return false; + return ((VespaVersion)other).versionNumber().equals(this.versionNumber()); + } + + public enum Confidence { + + /** This version has been proven defective */ + broken, + + /** We don't have sufficient evidence that this version is working */ + low, + + /** We have sufficient evidence that this version is working */ + normal, + + /** We have overwhelming evidence that this version is working */ + high + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java new file mode 100644 index 00000000000..f5852b9dfcf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.restapi.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.inject.Inject; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService; + +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.UriBuilder; + +/** + * Proxies requests from controller to https://xxx.statuspage.io/api/v2/yyy.json?api_key=zzz[&since=YYYY-MM-DDThh:mm[:ss]±hh:mm] + * + * @author andreer + */ +@Path("/v1/") +@Produces(MediaType.APPLICATION_JSON) +public class StatusPageResource implements com.yahoo.vespa.hosted.controller.api.statuspage.StatusPageResource { + + private final Client client; + private final KeyService keyService; + + @Inject + public StatusPageResource(@Component KeyService keyService) { + this(keyService, ClientBuilder.newClient()); + } + + protected StatusPageResource(KeyService keyService, Client client) { + this.keyService = keyService; + this.client = client; + } + + protected UriBuilder statusPageURL(String page, String since) { + String[] secrets = keyService.getSecret("vespa_hosted.controller.statuspage_api_key").split(":"); + UriBuilder uriBuilder = UriBuilder.fromUri("https://" + secrets[0] + ".statuspage.io/api/v2/" + page + ".json?api_key=" + secrets[1]); + if (since != null) { + uriBuilder.queryParam("since", since); + } + + return uriBuilder; + } + + @Override + public JsonNode statusPage(String page, String since) { + WebTarget target = client.target(statusPageURL(page, since)); + return target.request().get(JsonNode.class); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java new file mode 100644 index 00000000000..dca8a22a313 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author Tony Vaagenes + */ +@ExportPackage +package com.yahoo.vespa.hosted.restapi.impl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java new file mode 100644 index 00000000000..9eef1dac70b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java @@ -0,0 +1,148 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.rotation; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.log.LogLevel; +import com.yahoo.metrics.simple.Gauge; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; +import com.yahoo.vespa.hosted.controller.api.ApplicationAlias; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A rotation repository. + * + * @author Oyvind Gronnesby + */ +// TODO: Fold this into ApplicationController+Application +public class ControllerRotationRepository implements RotationRepository { + + private static final Logger log = Logger.getLogger(ControllerRotationRepository.class.getName()); + + private static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations"; + private final Gauge remainingRotations; + + private final ControllerDb controllerDb; + private final Map<RotationId, Rotation> rotationsMap; + + public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, MetricReceiver metricReceiver) { + this.controllerDb = controllerDb; + this.rotationsMap = buildRotationsMap(rotationConfig); + this.remainingRotations = metricReceiver.declareGauge(REMAINING_ROTATIONS_METRIC_NAME); + } + + private static Map<RotationId, Rotation> buildRotationsMap(RotationsConfig rotationConfig) { + return rotationConfig.rotations().entrySet().stream() + .map(entry -> { + RotationId rotationId = new RotationId(entry.getKey()); + return new Rotation(rotationId, entry.getValue().trim()); + }) + .collect(Collectors.toMap( + rotation -> rotation.rotationId, + rotation -> rotation + )); + } + + @Override + @NotNull + public Set<Rotation> getOrAssignRotation(ApplicationId applicationId, DeploymentSpec deploymentSpec) { + reportRemainingRotations(); + + Set<RotationId> rotations = controllerDb.getRotations(applicationId); + + if (rotations.size() > 1) { + log.warning(String.format("Application %s has %d > 1 rotation", applicationId, rotations.size())); + } + + if (!rotations.isEmpty()) { + return rotations.stream() + .map(rotationsMap::get) + .collect(Collectors.toSet()); + } + + if( ! deploymentSpec.globalServiceId().isPresent()) { + return Collections.emptySet(); + } + + long productionZoneCount = deploymentSpec.zones().stream() + .filter(zone -> zone.deploysTo(Environment.prod)) + .filter(zone -> ! isCorp(zone)) // Global rotations don't work for nodes in corp network + .count(); + + if (productionZoneCount >= 2) { + return assignRotation(applicationId); + } + else { + throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); + } + } + + private boolean isCorp(DeploymentSpec.DeclaredZone zone) { + return zone.region().isPresent() && zone.region().get().value().contains("corp"); + } + + @Override + @NotNull + public Set<URI> getRotationUris(ApplicationId applicationId) { + Set<RotationId> rotations = controllerDb.getRotations(applicationId); + if (rotations.isEmpty()) { + return Collections.emptySet(); + } + else { + ApplicationAlias applicationAlias = new ApplicationAlias(applicationId); + return Collections.singleton(applicationAlias.toHttpUri()); + } + } + + private Set<Rotation> assignRotation(ApplicationId applicationId) { + Set<RotationId> availableRotations = availableRotations(); + if (availableRotations.isEmpty()) { + String message = "Unable to assign global rotation to " + + applicationId + " - no rotations available"; + log.info(message); + throw new RuntimeException(message); + } + + for (RotationId rotationId : availableRotations) { + if (controllerDb.assignRotation(rotationId, applicationId)) { + log.info(String.format("Assigned rotation %s to application %s", rotationId, applicationId)); + Rotation rotation = this.rotationsMap.get(rotationId); + return Collections.singleton(rotation); + } + } + + log.info(String.format("Rotation: No rotations assigned with %s rotations available", availableRotations.size())); + return Collections.emptySet(); + } + + private Set<RotationId> availableRotations() { + Set<RotationId> assignedRotations = controllerDb.getRotations(); + Set<RotationId> allRotations = new HashSet<>(rotationsMap.keySet()); + allRotations.removeAll(assignedRotations); + return allRotations; + } + + private void reportRemainingRotations() { + try { + int freeRotationsCount = availableRotations().size(); + log.log(LogLevel.INFO, "Rotation: {0} global rotations remaining", freeRotationsCount); + remainingRotations.sample(freeRotationsCount); + } catch (Exception e) { + log.log(LogLevel.INFO, "Failed to report rotations metric", e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java new file mode 100644 index 00000000000..4e333f0268b --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java @@ -0,0 +1,54 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.rotation; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * A rotation repository backed by in-memory data structures + * + * @author bratseth + */ +public class MemoryRotationRepository implements RotationRepository { + + private final Map<ApplicationId, Set<Rotation>> rotations = new HashMap<>(); + + @NotNull + @Override + public Set<Rotation> getOrAssignRotation(ApplicationId application, DeploymentSpec deploymentSpec) { + if (rotations.containsKey(application)) { + return rotations.get(application); + } + Set<Rotation> rotations = ImmutableSet.of(new Rotation( + new RotationId("generated-by-routing-service-" + UUID.randomUUID().toString()), + "fake-global-rotation-" + application.toShortString()) + ); + this.rotations.put(application, rotations); + return rotations; + } + + @NotNull + @Override + public Set<URI> getRotationUris(ApplicationId applicationId) { + Set<Rotation> rotations = this.rotations.get(applicationId); + if (rotations == null) { + return Collections.emptySet(); + } + return rotations.stream() + .map(rotation -> URI.create("http://" + rotation.rotationName)) + .collect(Collectors.toSet()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java new file mode 100644 index 00000000000..b1f7b33e58e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java @@ -0,0 +1,48 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.rotation; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.Set; + +/** + * A rotation repository assigns global rotations to Vespa applications. It does not take into account + * whether an application qualifies or not, but it assumes that each application should get only + * one. + * + * The list of rotations comes from the RotationsConfig, set in the controller's services.xml. + * Assignments are persisted with the RotationId as the primary key. When we assign the + * rotation to an application we try to put the mapping RotationId -> Application. If a + * mapping already exists for that RotationId, the assignment will fail. + * + * @author Oyvind Gronnesby + */ +public interface RotationRepository { + + // TODO: Change to use provision.ApplicationId + // TODO: Move the persistence into ControllerDb (done), and then collapse the 2 implementations and the interface into one + + /** + * If any rotations are assigned to the application, these will be returned. + * If no rotations are assigned, assign one rotation to the application and return that. + * + * @param applicationId ID of the application to get or assign rotation for + * @param deploymentSpec Spec of current application being deployed + * @return Set of rotations assigned (may be empty) + */ + @NotNull + Set<Rotation> getOrAssignRotation(ApplicationId applicationId, DeploymentSpec deploymentSpec); + + /** + * Get the external visible rotation URIs for this application. + * + * @param applicationId ID of the application to get or assign rotation for + */ + @NotNull + Set<URI> getRotationUris(ApplicationId applicationId); + +} diff --git a/controller-server/src/main/resources/WEB-INF/web.xml b/controller-server/src/main/resources/WEB-INF/web.xml new file mode 100644 index 00000000000..f294a8eb46e --- /dev/null +++ b/controller-server/src/main/resources/WEB-INF/web.xml @@ -0,0 +1,8 @@ +<?xml version="1.0"?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<jdisc-config> + <init-param> + <param-name>com.sun.jersey.spi.container.ContainerRequestFilters</param-name> + <param-value>com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext.PropagateSecurityContextFilter</param-value> + </init-param> +</jdisc-config> diff --git a/controller-server/src/main/resources/configdefinitions/http-access-control.def b/controller-server/src/main/resources/configdefinitions/http-access-control.def new file mode 100644 index 00000000000..4cd1532761b --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/http-access-control.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.restapi.filter.config + +allowedUrls[] string diff --git a/controller-server/src/main/resources/configdefinitions/maintainer.def b/controller-server/src/main/resources/configdefinitions/maintainer.def new file mode 100644 index 00000000000..7ec8860bef4 --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/maintainer.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.maintenance.config + +intervalMinutes int default=30 diff --git a/controller-server/src/main/resources/configdefinitions/rotations.def b/controller-server/src/main/resources/configdefinitions/rotations.def new file mode 100644 index 00000000000..d4f3636d0d8 --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/rotations.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.rotation.config + +rotations{} string diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java new file mode 100644 index 00000000000..6018c99206e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -0,0 +1,204 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.ClusterView; +import com.yahoo.vespa.serviceview.bindings.ServiceView; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +/** + * @author mortent + */ +public class ConfigServerClientMock extends AbstractComponent implements ConfigServerClient { + + private Map<ApplicationId, byte[]> applicationContent = new HashMap<>(); + private Map<ApplicationId, String> applicationInstances = new HashMap<>(); + private Map<ApplicationId, Boolean> applicationActivated = new HashMap<>(); + private Set<ApplicationId> applicationRestarted = new HashSet<>(); + private Set<String> hostsExplicitlyRestarted = new HashSet<>(); + private Map<String, EndpointStatus> endpoints = new HashMap<>(); + + private Map<URI, Version> configServerVersions = new HashMap<>(); + private Version defaultConfigServerVersion = new Version(6, 1, 0); + + /** The exception to throw on the next prepare run, or null to continue normally */ + private RuntimeException prepareException = null; + + /** The version given in the previous prepare call, or null if no call has been made */ + public Optional<Version> lastPrepareVersion = null; + + @Override + public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) { + lastPrepareVersion = deployOptions.vespaVersion.map(Version::new); + + if (prepareException != null) { + RuntimeException prepareException = this.prepareException; + this.prepareException = null; + throw prepareException; + } + + applicationContent.put(deployment.applicationId(), content); + applicationActivated.put(deployment.applicationId(), false); + applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080"); + + return new PreparedApplication() { + @Override + public void activate() { + applicationActivated.put(deployment.applicationId(), true); + } + + @Override + public List<Log> messages() { + Log warning = new Log(); + warning.level = "WARNING"; + warning.time = 1; + warning.message = "The warning"; + + Log info = new Log(); + info.level = "INFO"; + info.time = 2; + info.message = "The info"; + + return Arrays.asList(warning, info); + } + + @Override + public PrepareResponse prepareResponse() { + PrepareResponse prepareResponse = new PrepareResponse(); + prepareResponse.message = "foo"; + prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); + prepareResponse.tenant = new TenantId("tenant"); + return prepareResponse; + } + }; + } + + public void throwOnNextPrepare(RuntimeException prepareException) { + this.prepareException = prepareException; + } + + /** + * Returns the (initially empty) mutable map of config server urls to versions. + * This API will return defaultConfigserverVersion as response to any version(url) call for versions not added to the map. + */ + public Map<URI, Version> configServerVersions() { + return configServerVersions; + } + + public Version getDefaultConfigServerVersion() { return defaultConfigServerVersion; } + public void setDefaultConfigServerVersion(Version version) { defaultConfigServerVersion = version; } + + @Override + public List<String> getNodeQueryHost(DeploymentId deployment, String type) { + if (applicationInstances.containsKey(deployment.applicationId())) { + return Collections.singletonList(applicationInstances.get(deployment.applicationId())); + } else { + return Collections.emptyList(); + } + } + + @Override + public void restart(DeploymentId deployment, Optional<Hostname> hostname) { + applicationRestarted.add(deployment.applicationId()); + if (hostname.isPresent()) { + hostsExplicitlyRestarted.add(hostname.get().id()); + } + } + + @Override + public void deactivate(DeploymentId deployment) { + applicationActivated.remove(deployment.applicationId()); + applicationContent.remove(deployment.applicationId()); + applicationInstances.remove(deployment.applicationId()); + } + + @Override + public JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds) { + ObjectNode root = new ObjectNode(JsonNodeFactory.instance); + root.put("generation", 1); + return root; + } + + @Override + public JsonNode grabLog(DeploymentId applicationInstance) { + return new ObjectNode(JsonNodeFactory.instance); + } + + // Returns a canned example response + @Override + public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { + ApplicationView applicationView = new ApplicationView(); + ClusterView cluster = new ClusterView(); + cluster.name = "cluster1"; + cluster.type = "content"; + cluster.url = "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1"; + ServiceView service = new ServiceView(); + service.configId = "cluster1/storage/0"; + service.host = "host1"; + service.serviceName = "storagenode"; + service.serviceType = "storagenode"; + service.url = "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/"; + cluster.services = new ArrayList<>(); + cluster.services.add(service); + applicationView.clusters = new ArrayList<>(); + applicationView.clusters.add(cluster); + return applicationView; + } + + // Returns a canned example response + @Override + public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath) { + Map<String,List<?>> root = new HashMap<>(); + List<Map<?,?>> resources = new ArrayList<>(); + Map<String,String> resource = new HashMap<>(); + resource.put("url", "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config"); + resources.add(resource); + root.put("resources", resources); + return root; + } + + @Override + public Version version(URI configServerURI) { + return configServerVersions.getOrDefault(configServerURI, defaultConfigServerVersion); + } + + @Override + public void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status) { + endpoints.put(endpoint, status); + } + + @Override + public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) { + EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757l); + return endpoints.containsKey(endpoint) + ? endpoints.get(endpoint) + : result; + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java new file mode 100644 index 00000000000..e807762371a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -0,0 +1,601 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.component.Version; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; +import org.junit.Test; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static org.fest.assertions.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author bratseth + * @author mpolden + */ +public class ControllerTest { + + private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + + @Test + public void testDeployment() { + // Setup system + DeploymentTester tester = new DeploymentTester(); + ApplicationController applications = tester.controller().applications(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-east-3") + .build(); + + // staging job - succeeding + Version version1 = Version.fromString("6.1"); // Set in config server mock + Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); + applications.notifyJobCompletion(mockReport(app1, component, true, false)); + assertFalse("Revision is currently not known", + ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); + + Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision(); + assertTrue("Revision has been set during deployment", revision.isPresent()); + assertStatus(JobStatus.initial(stagingTest) + .withTriggering(version1, revision, tester.clock().instant()) + .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + + // Causes first deployment job to be triggered + assertStatus(JobStatus.initial(productionCorpUsEast1) + .withTriggering(version1, revision, tester.clock().instant()), app1.id(), tester.controller()); + tester.clock().advance(Duration.ofSeconds(1)); + + // production job (failing) + tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, false); + assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); + + JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) + .withTriggering(version1, revision, tester.clock().instant()) // Triggered first without revision info + .withCompletion(Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) + .withTriggering(version1, revision, tester.clock().instant()); // Re-triggering (due to failure) has revision info + + assertStatus(expectedJobStatus, app1.id(), tester.controller()); + + // Simulate restart + tester.restartController(); + applications = tester.controller().applications(); + + assertNotNull(tester.controller().tenants().tenant(new TenantId("tenant1"))); + assertNotNull(applications.get(ApplicationId.from(TenantName.from("tenant1"), + ApplicationName.from("application1"), + InstanceName.from("default")))); + assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); + + tester.clock().advance(Duration.ofSeconds(1)); + + // system and staging test job - succeeding + applications.notifyJobCompletion(mockReport(app1, component, true, false)); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + assertStatus(JobStatus.initial(systemTest) + .withTriggering(version1, revision, tester.clock().instant()) + .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + + // production job succeeding now + tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, true); + expectedJobStatus = expectedJobStatus + .withTriggering(version1, revision, tester.clock().instant()) + .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()); + assertStatus(expectedJobStatus, app1.id(), tester.controller()); + + // causes triggering of next production job + assertStatus(JobStatus.initial(productionUsEast3) + .withTriggering( version1, revision, tester.clock().instant()), + app1.id(), tester.controller()); + tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true); + + assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size()); + + // prod zone removal is not allowed + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-east-3") + .build(); + applications.notifyJobCompletion(mockReport(app1, component, true, false)); + try { + tester.deploy(systemTest, app1, applicationPackage); + fail("Expected exception due to unallowed production deployment removal"); + } + catch (IllegalArgumentException e) { + assertEquals("deployment-removal: application 'tenant1.app1' is deployed in corp-us-east-1, but does not include this zone in deployment.xml", e.getMessage()); + } + assertNotNull("Zone was not removed", + applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); + assertNotNull("Deployment job was not removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); + + // prod zone removal is allowed with override + applicationPackage = new ApplicationPackageBuilder() + .allow(ValidationId.deploymentRemoval) + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-east-3") + .build(); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + assertNull("Zone was removed", + applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); + assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); + } + + @Test + public void testDeployVersion() { + // Setup system + DeploymentTester tester = new DeploymentTester(); + ApplicationController applications = tester.controller().applications(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .build(); + Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber(); + + Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); + applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(false)), applications.lock(app1.id())); + + // First deployment: An application change + applications.notifyJobCompletion(mockReport(app1, component, true, false)); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true); + + app1 = applications.require(app1.id()); + assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); + assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + + // Unexpected deployment + try { + tester.deploy(productionUsWest1, app1, applicationPackage); + fail("Expected exception as no change was to be deployed"); + } + catch (IllegalArgumentException expected) { + // success + } + + // Application change after a new system version, and a region added + Version newSystemVersion = incrementSystemVersion(tester.controller()); + assertTrue(newSystemVersion.isAfter(systemVersion)); + + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-east-3") + .build(); + applications.notifyJobCompletion(mockReport(app1, component, true, false)); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true); + + app1 = applications.require(app1.id()); + assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); + assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + + // A deployment to the new region gets the same version + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-east-3") + .build(); + tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true); + app1 = applications.require(app1.id()); + assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); + assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + + // Version upgrade changes system version + Change.VersionChange change = new Change.VersionChange(newSystemVersion); + applications.deploymentTrigger().triggerChange(app1.id(), change); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(stagingTest, app1, applicationPackage, true); + tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true); + tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true); + + app1 = applications.require(app1.id()); + assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); + assertEquals(newSystemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + } + + /** Adds a new version, higher than the current system version, makes it the system version and returns it */ + private Version incrementSystemVersion(Controller controller) { + Version systemVersion = controller.versionStatus().systemVersion().get().versionNumber(); + Version newSystemVersion = new Version(systemVersion.getMajor(), systemVersion.getMinor()+1, 0); + VespaVersion newSystemVespaVersion = new VespaVersion(DeploymentStatistics.empty(newSystemVersion), + "commit1", + Instant.now(), + true, + Collections.emptyList(), + controller); + List<VespaVersion> versions = new ArrayList<>(controller.versionStatus().versions()); + for (int i = 0; i < versions.size(); i++) { + VespaVersion c = versions.get(i); + if (c.isCurrentSystemVersion()) + versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), false, c.configServerHostnames(), controller)); + } + versions.add(newSystemVespaVersion); + controller.updateVersionStatus(new VersionStatus(versions)); + return newSystemVersion; + } + + @Test + public void testPullRequestDeployment() { + // Setup system + ControllerTester tester = new ControllerTester(); + ApplicationController applications = tester.controller().applications(); + + // staging deployment + long app1ProjectId = 22; + ApplicationId app1 = tester.createAndDeploy("tenant1", "domain1", "application1", Environment.staging, app1ProjectId).id(); + + // pull-request deployment - uses different instance id + ApplicationId app1pr = tester.createAndDeploy("tenant1", "domain1", "application1", "default-pr1", Environment.staging, app1ProjectId, null).id(); + + assertTrue(applications.get(app1).isPresent()); + assertEquals(app1, applications.get(app1).get().id()); + assertTrue(applications.get(app1pr).isPresent()); + assertEquals(app1pr, applications.get(app1pr).get().id()); + + // Simulate restart + tester.createNewController(); + applications = tester.controller().applications(); + + assertTrue(applications.get(app1).isPresent()); + assertEquals(app1, applications.get(app1).get().id()); + assertTrue(applications.get(app1pr).isPresent()); + assertEquals(app1pr, applications.get(app1pr).get().id()); + } + + @Test + public void testFailingSinceUpdates() { + // Setup system + DeploymentTester tester = new DeploymentTester(); + + // Setup application + Application app = tester.createApplication("app1", "foo", 1, 1L); + + // Initial failure + Instant initialFailure = tester.clock().instant(); + tester.notifyJobCompletion(component, app, true); + tester.deployAndNotify(systemTest, app, applicationPackage, false); + assertEquals("Failure age is right at initial failure", + initialFailure, firstFailing(app, tester).get().at()); + + // Failure again -- failingSince should remain the same + tester.clock().advance(Duration.ofMillis(1000)); + tester.deployAndNotify(systemTest, app, applicationPackage, false); + assertEquals("Failure age is right at second consecutive failure", + initialFailure, firstFailing(app, tester).get().at()); + + // Success resets failingSince + tester.clock().advance(Duration.ofMillis(1000)); + tester.deployAndNotify(systemTest, app, applicationPackage, true); + assertFalse(firstFailing(app, tester).isPresent()); + + // Complete deployment + tester.deployAndNotify(stagingTest, app, applicationPackage, true); + tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true); + + // Two repeated failures again. + // Initial failure + tester.clock().advance(Duration.ofMillis(1000)); + initialFailure = tester.clock().instant(); + tester.notifyJobCompletion(component, app, true); + tester.deployAndNotify(systemTest, app, applicationPackage, false); + assertEquals("Failure age is right at initial failure", + initialFailure, firstFailing(app, tester).get().at()); + + // Failure again -- failingSince should remain the same + tester.clock().advance(Duration.ofMillis(1000)); + tester.deployAndNotify(systemTest, app, applicationPackage, false); + assertEquals("Failure age is right at second consecutive failure", + initialFailure, firstFailing(app, tester).get().at()); + } + + private Optional<JobStatus.JobRun> firstFailing(Application application, DeploymentTester tester) { + return tester.controller().applications().get(application.id()).get().deploymentJobs().jobStatus().get(systemTest).firstFailing(); + } + + @Test + public void testMigratingTenantToAthensWillModifyAthensDomainsCorrectly() { + ControllerTester tester = new ControllerTester(); + + // Create Athens domain mock + AthensDomain athensDomain = new AthensDomain("vespa.john"); + AthensDbMock.Domain mockDomain = new AthensDbMock.Domain(athensDomain); + tester.athensDb().addDomain(mockDomain); + + // Create OpsDb tenant + TenantId tenantId = new TenantId("mytenant"); + Tenant existingTenant = Tenant.createOpsDbTenant(tenantId, new UserGroup("myusergroup"), new Property("myproperty")); + tester.controller().tenants().addTenant(existingTenant, Optional.empty()); + + // Create an application without instance + String applicationName = "myapplication"; + ApplicationId applicationId = ApplicationId.from(tenantId.id(), applicationName, "default"); + tester.controller().applications().createApplication(applicationId, Optional.empty()); + + // Verify that Athens domain does not have any relations to tenant/application yet + assertThat(mockDomain.applications.keySet()).isEmpty(); + assertThat(mockDomain.isVespaTenant).isFalse(); + + // Migrate tenant to Athens + NToken nToken = new NTokenMock("token"); + tester.controller().tenants().migrateTenantToAthens( + tenantId, athensDomain, new PropertyId("1567"), new Property("vespa_dev.no"), nToken); + + // Verify that tenant is migrated + Tenant tenant = tester.controller().tenants().tenant(tenantId).get(); + assertThat(tenant.isAthensTenant()) + .isTrue(); + assertThat(tenant.getAthensDomain().get()) + .isEqualTo(athensDomain); + // Verify that domain knows about tenant and application + assertThat(mockDomain.isVespaTenant) + .isTrue(); + assertThat(mockDomain.applications.keySet()) + .contains(new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationName)); + } + + @Test + public void selfTriggeringApplicationIsNotTriggered() { + ControllerTester tester = new ControllerTester(); + ApplicationController applications = tester.controller().applications(); + + // Create application and report completion from component job + long projectId = 1; + TenantId tenant = tester.createTenant("tenant", "domain", 1L); + Application application = tester.createApplication(tenant, "application", "default", projectId); + applications.notifyJobCompletion(mockReport(application, component, true, true)); + + // Only component completion status is persisted and no further jobs are triggered + assertEquals(1, applications.get(application.id()).get().deploymentJobs().jobStatus().size()); + assertStatus(JobStatus.initial(component).withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), + application.id(), tester.controller()); + } + + @Test + public void requeueOutOfCapacityStagingJob() { + DeploymentTester tester = new DeploymentTester(); + + long fooProjectId = 1; + long barProjectId = 2; + Application foo = tester.createApplication("app1", "foo", fooProjectId, 1L); + Application bar = tester.createApplication("app2", "bar", barProjectId, 1L); + BuildSystem buildSystem = tester.controller().applications().deploymentTrigger().buildSystem(); + + // foo: passes system test + tester.notifyJobCompletion(component, foo, true); + tester.deployAndNotify(systemTest, foo, applicationPackage, true); + + // bar: passes system test + tester.notifyJobCompletion(component, bar, true); + tester.deployAndNotify(systemTest, bar, applicationPackage, true); + + // foo and bar: staging test jobs queued + assertEquals(2, buildSystem.jobs().size()); + + // foo: staging-test job fails with out of capacity and is added to the front of the queue + { + tester.deploy(stagingTest, foo, applicationPackage); + tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity)); + List<BuildJob> nextJobs = buildSystem.takeJobsToRun(); + assertEquals("staging-test jobs are returned one at a time",1, nextJobs.size()); + assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(fooProjectId, nextJobs.get(0).projectId()); + } + + // bar: Completes deployment + tester.deployAndNotify(stagingTest, bar, applicationPackage, true); + tester.deployAndNotify(productionCorpUsEast1, bar, applicationPackage, true); + + // foo: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by + // out of capacity retry mechanism + tester.clock().advance(Duration.ofMinutes(15)); + tester.notifyJobCompletion(component, foo, true); + tester.deployAndNotify(systemTest, foo, applicationPackage, true); + tester.deploy(stagingTest, foo, applicationPackage); + assertEquals(1, buildSystem.takeJobsToRun().size()); + tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity)); + assertTrue("No jobs queued", buildSystem.jobs().isEmpty()); + + // bar: New change triggers another staging-test job + tester.notifyJobCompletion(component, bar, true); + tester.deployAndNotify(systemTest, bar, applicationPackage, true); + assertEquals(1, buildSystem.jobs().size()); + + // foo: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the + // back of the queue + tester.clock().advance(Duration.ofHours(3)); + tester.clock().advance(Duration.ofMinutes(50)); + tester.failureRedeployer().maintain(); + + List<BuildJob> nextJobs = buildSystem.takeJobsToRun(); + assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(barProjectId, nextJobs.get(0).projectId()); + nextJobs = buildSystem.takeJobsToRun(); + assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(fooProjectId, nextJobs.get(0).projectId()); + } + + private void assertStatus(JobStatus expectedStatus, ApplicationId id, Controller controller) { + Application app = controller.applications().get(id).get(); + JobStatus existingStatus = app.deploymentJobs().jobStatus().get(expectedStatus.type()); + assertNotNull("Status of type " + expectedStatus.type() + " is present", existingStatus); + assertEquals(expectedStatus, existingStatus); + } + + private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError, boolean selfTriggering) { + return new JobReport( + application.id(), + jobType, + application.deploymentJobs().projectId().get(), + 1L, + jobError, + selfTriggering, + true + ); + } + + private JobReport mockReport(Application application, JobType jobType, boolean success, boolean selfTriggering) { + return mockReport(application, jobType, JobError.from(success), selfTriggering); + } + + @Test + public void testGlobalRotations() throws IOException { + // Setup tester and app def + ControllerTester tester = new ControllerTester(); + Zone zone = Zone.defaultZone(); + ApplicationId appId = tester.applicationId("tenant", "app1", "default"); + DeploymentId deployId = new DeploymentId(appId, zone); + + // Check initial rotation status + Map<String, EndpointStatus> rotationStatus = tester.controller().applications().getGlobalRotationStatus(deployId); + assertEquals(2, rotationStatus.size()); + + assertTrue(rotationStatus.get("global-endpoint").getStatus().equals(EndpointStatus.Status.in)); + assertTrue(rotationStatus.get("alias-endpoint").getStatus().equals(EndpointStatus.Status.in)); + + // Set the global rotations out of service + EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "Testing I said", "Test", tester.clock().instant().getEpochSecond()); + List<String> overrides = tester.controller().applications().setGlobalRotationStatus(deployId, status); + assertEquals(2, overrides.size()); + + // Recheck the override rotation status + rotationStatus = tester.controller().applications().getGlobalRotationStatus(deployId); + assertEquals(2, rotationStatus.size()); + assertTrue(rotationStatus.get("global-endpoint").getStatus().equals(EndpointStatus.Status.out)); + assertTrue(rotationStatus.get("alias-endpoint").getStatus().equals(EndpointStatus.Status.out)); + assertTrue(rotationStatus.get("alias-endpoint").getReason().equals("Testing I said")); + } + + @Test + public void testLegacyDeployments() { + // Setup system + DeploymentTester tester = new DeploymentTester(); + ApplicationController applications = tester.controller().applications(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-east-3") + .build(); + Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber(); + + Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); + applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(true)), applications.lock(app1.id())); + + // Scenario: App already on 6.0, Upgrade to 6.1 (systemversion) + Zone prodZone = new Zone(Environment.prod, RegionName.from("us-east-3")); + Zone stagingZone = new Zone(Environment.staging, RegionName.from("us-east-3")); + Version existingVersion = Version.fromString("6.0"); + + // Add deployment on existing version + legacyDeploy(tester.controller(), app1, applicationPackage, prodZone, Optional.of(existingVersion), false); + + // Add dev/perf deployment on old version to verify that this does not affect Initialize staging step. VESPA-8469 + Version devVersion = Version.fromString("5.0"); + legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.dev, RegionName.from("us-east-1")), Optional.of(devVersion), false); + legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.perf, RegionName.from("us-east-3")), Optional.of(devVersion), false); + + // Initialize staging on existing version + legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), true); + app1 = applications.require(app1.id()); + assertEquals(existingVersion, app1.currentDeployVersion(tester.controller(), stagingZone)); + + // Upgrade to the new version in staging + legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), false); + app1 = applications.require(app1.id()); + assertEquals(systemVersion, app1.currentDeployVersion(tester.controller(), stagingZone)); + } + + @Test + public void testDeployUntestedChangeFails() { + ControllerTester tester = new ControllerTester(); + ApplicationController applications = tester.controller().applications();TenantId tenant = tester.createTenant("tenant1", "domain1", 11L); + Application app = tester.createApplication(tenant, "app1", "default", 1); + + app = app.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.3")))) + .with(app.deploymentJobs().asSelfTriggering(false)); + applications.store(app, applications.lock(app.id())); + try { + tester.deploy(app, new Zone(Environment.prod, RegionName.from("us-east-3"))); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as pending version change to 6.3 is untested", e.getMessage()); + } + } + + private void legacyDeploy(Controller controller, Application application, ApplicationPackage applicationPackage, Zone zone, Optional<Version> version, boolean deployCurrentVersion) { + ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get())); + GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); + controller.applications().deployApplication(application.id(), + zone, + applicationPackage, + new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), version, false, deployCurrentVersion)); + + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java new file mode 100644 index 00000000000..41d9bbea5b2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -0,0 +1,202 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; +import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; +import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; +import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; +import com.yahoo.vespa.hosted.controller.api.integration.jira.JiraMock; +import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.cost.CostMock; +import com.yahoo.vespa.hosted.controller.cost.MockInsightBackend; +import com.yahoo.vespa.hosted.controller.integration.MockMetricsService; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.routing.MockRoutingGenerator; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.rotation.MemoryRotationRepository; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; + +import java.util.Optional; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Convenience methods for controller tests. + * This completely wraps TestEnvironment to make it easier to get rid of that in the future. + * + * @author bratseth + */ +public final class ControllerTester { + + private final ControllerDb db = new MemoryControllerDb(); + private final AthensDbMock athensDb = new AthensDbMock(); + private final ManualClock clock = new ManualClock(); + private final ConfigServerClientMock configServerClientMock = new ConfigServerClientMock(); + private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(); + private final GitHubMock gitHubMock = new GitHubMock(); + private final CuratorDb curator = new MockCuratorDb(); + private Controller controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, athensDb); + + private static final Controller createController(ControllerDb db, CuratorDb curator, + ConfigServerClientMock configServerClientMock, ManualClock clock, + GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, + AthensDbMock athensDb) { + Controller controller = new Controller(db, + curator, + new MemoryRotationRepository(), + gitHubClientMock, + new JiraMock(), + new MemoryEntityService(), + new MemoryGlobalRoutingService(), + zoneRegistryMock, + new CostMock(new MockInsightBackend()), + configServerClientMock, + new MockMetricsService(), + new MemoryNameService(), + new MockRoutingGenerator(), + new ChefMock(), + clock, + new AthensMock(athensDb)); + controller.updateVersionStatus(VersionStatus.compute(controller)); + return controller; + } + + public Controller controller() { return controller; } + public CuratorDb curator() { return curator; } + public ManualClock clock() { return clock; } + public AthensDbMock athensDb() { return athensDb; } + + /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ + public final void createNewController() { + controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, athensDb); + } + + public ZoneRegistryMock getZoneRegistryMock() { return zoneRegistryMock; } + + public ConfigServerClientMock configServerClientMock() { return configServerClientMock; } + + public GitHubMock gitHubClientMock () { return gitHubMock; } + + /** Set the application with the given id to currently be in the progress of rolling out the given change */ + public void setDeploying(ApplicationId id, Optional<Change> change) { + try (Lock lock = controller.applications().lock(id)) { + controller.applications().store(controller.applications().require(id).withDeploying(change), lock); + } + } + + /** Creates the given tenant and application and deploys it */ + public Application createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId, Long propertyId) { + return createAndDeploy(tenantName, domainName, applicationName, toZone(environment), projectId, propertyId); + } + + /** Creates the given tenant and application and deploys it */ + public Application createAndDeploy(String tenantName, String domainName, String applicationName, + String instanceName, Zone zone, long projectId, Long propertyId) { + TenantId tenant = createTenant(tenantName, domainName, propertyId); + Application application = createApplication(tenant, applicationName, instanceName, projectId); + deploy(application, zone); + return application; + } + + /** Creates the given tenant and application and deploys it */ + public Application createAndDeploy(String tenantName, String domainName, String applicationName, + String instanceName, Environment environment, long projectId, Long propertyId) { + return createAndDeploy(tenantName, domainName, applicationName, instanceName, toZone(environment), projectId, propertyId); + } + + /** Creates the given tenant and application and deploys it */ + public Application createAndDeploy(String tenantName, String domainName, String applicationName, Zone zone, long projectId, Long propertyId) { + return createAndDeploy(tenantName, domainName, applicationName, "default", zone, projectId, propertyId); + } + + /** Creates the given tenant and application and deploys it */ + public Application createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId) { + return createAndDeploy(tenantName, domainName, applicationName, environment, projectId, null); + } + + public Zone toZone(Environment environment) { + switch (environment) { + case dev: case test: return new Zone(environment, RegionName.from("us-east-1")); + case staging: return new Zone(environment, RegionName.from("us-east-3")); + default: return new Zone(environment, RegionName.from("us-west-1")); + } + } + + public AthensDomain createDomain(String domainName) { + AthensDomain domain = new AthensDomain(domainName); + athensDb.addDomain(new AthensDbMock.Domain(domain)); + return domain; + } + + public TenantId createTenant(String tenantName, String domainName, Long propertyId) { + TenantId id = new TenantId(tenantName); + Optional<Tenant> existing = controller().tenants().tenant(id); + if (existing.isPresent()) return id; + + Tenant tenant = Tenant.createAthensTenant(id, createDomain(domainName), new Property("app1Property"), + propertyId == null ? Optional.empty() : Optional.of(new PropertyId(propertyId.toString()))); + controller().tenants().addTenant(tenant, Optional.of(TestIdentities.userNToken)); + assertNotNull(controller().tenants().tenant(id)); + return id; + } + + public Application createApplication(TenantId tenant, String applicationName, String instanceName, long projectId) { + ApplicationId applicationId = applicationId(tenant.id(), applicationName, instanceName); + Application application = controller().applications().createApplication(applicationId, Optional.of(TestIdentities.userNToken)) + .withProjectId(projectId); + assertTrue(controller().applications().get(applicationId).isPresent()); + return application; + } + + public void deploy(Application application, Zone zone) { + deploy(application, zone, new ApplicationPackage(new byte[0])); + } + + public void deploy(Application application, Zone zone, ApplicationPackage applicationPackage) { + deploy(application, zone, applicationPackage, false); + } + + public void deploy(Application application, Zone zone, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { + ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get())); + GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); + controller().applications().deployApplication(application.id(), + zone, + applicationPackage, + new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), Optional.empty(), false, deployCurrentVersion)); + } + + public ApplicationId applicationId(String tenant, String application, String instance) { + return ApplicationId.from(TenantName.from(tenant), + ApplicationName.from(application), + InstanceName.from(instance)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java new file mode 100644 index 00000000000..343a9d2ed6e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java @@ -0,0 +1,83 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + + +import com.yahoo.jdisc.Metric; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class MetricsMock implements Metric { + + private final Map<Context, Map<String, Number>> metrics = new HashMap<>(); + + @Override + public void set(String key, Number val, Context ctx) { + Map<String, Number> metricsMap = metrics.getOrDefault(ctx, new HashMap<>()); + metricsMap.put(key, val); + } + + @Override + public void add(String key, Number val, Context ctx) { + Map<String, Number> metricsMap = metrics.getOrDefault(ctx, new HashMap<>()); + metricsMap.compute(key, (k, v) -> v == null ? val : sum(v, val)); + } + + private Number sum(Number n1, Number n2) { + return n1.doubleValue() + n2.doubleValue(); + } + + @Override + public Context createContext(Map<String, ?> properties) { + Context ctx = new MapContext(properties); + metrics.putIfAbsent(ctx, new HashMap<>()); + return ctx; + } + + public Map<Context, Map<String, Number>> getMetrics() { + return metrics; + } + + /** Returns a zero-context metric by name, or null if it is not present */ + public Number getMetric(String name) { + Map<String, Number> valuesForEmptyContext = metrics.get(createContext(Collections.emptyMap())); + if (valuesForEmptyContext == null) return null; + return valuesForEmptyContext.get(name); + } + + public Map<MapContext, Map<String, Number>> getMetricsFilteredByHost(String hostname) { + return getMetrics().entrySet().stream() + .filter(entry -> ((MapContext)entry.getKey()).containsDimensionValue("host", hostname)) + .collect(Collectors.toMap(entry -> (MapContext) entry.getKey(), Map.Entry::getValue)); + } + + public static class MapContext implements Context { + final Map<String, ?> dimensions; + + public MapContext(Map<String, ?> dimensions) { + this.dimensions = dimensions; + } + + @Override + public boolean equals(Object obj) { + return Objects.deepEquals(obj, dimensions); + } + + @Override + public int hashCode() { + return Objects.toString(dimensions).hashCode(); + } + + public Map<String, ?> getDimensions() { + return dimensions; + } + + public boolean containsDimensionValue(String dimension, Object value) { + return value.equals(dimensions.get(dimension)); + } + } +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java new file mode 100644 index 00000000000..1f52ebcadb7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; + +/** + * @author Tony Vaagenes + */ +public class TestIdentities { + + public static UserId userId = new UserId("mytenant"); + + public static TenantId tenantId = new TenantId("mynonusertenant"); + + public static EnvironmentId environment = new EnvironmentId("dev"); + + public static RegionId region = new RegionId("us-east-1"); + + public static InstanceId instance = new InstanceId("default"); + + public static UserGroup userGroup1 = new UserGroup("usergroup1"); + + public static Property property = new Property("property"); + + public static Tenant tenant = Tenant.createOpsDbTenant(tenantId, userGroup1, property); + + public static NToken userNToken = new NTokenMock("token"); + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java new file mode 100644 index 00000000000..62b935842f7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -0,0 +1,74 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; + +import java.net.URI; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * @author mpolden + */ +public class ZoneRegistryMock implements ZoneRegistry { + + private final Map<Zone, Duration> deploymentTimeToLive = new HashMap<>(); + + public void setDeploymentTimeToLive(Zone zone, Duration duration) { + deploymentTimeToLive.put(zone, duration); + } + + @Override + public SystemName system() { + return SystemName.main; + } + + @Override + public List<Zone> zones() { + return Collections.singletonList(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("corp-us-east-1"))); + } + + @Override + public Optional<Zone> getZone(Environment environment, RegionName region) { + return zones().stream().filter(z -> z.environment().equals(environment) && z.region().equals(region)).findFirst(); + } + + @Override + public List<URI> getConfigServerUris(Environment environment, RegionName region) { + return getZone(environment, region) + .map(z -> URI.create(String.format("http://cfg.%s.%s.test", environment.value(), region.value()))) + .map(Collections::singletonList) + .orElse(Collections.emptyList()); + } + + @Override + public Optional<URI> getLogServerUri(Environment environment, RegionName region) { + return getZone(environment, region) + .map(z -> URI.create(String.format("http://log.%s.%s.test", environment.value(), region.value()))); + } + + @Override + public Optional<Duration> getDeploymentTimeToLive(Environment environment, RegionName region) { + return Optional.ofNullable(deploymentTimeToLive.get(new Zone(environment, region))); + } + + @Override + public URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application) { + return URI.create("http://monitoring-system.test/?environment=" + environment.value() + "®ion=" + + name.value() + "&application=" + application.toShortString()); + } + + @Override + public URI getDashboardUri() { + return URI.create("http://dashboard.test"); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java new file mode 100644 index 00000000000..0a5ddfb5efc --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.cost; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; +import com.yahoo.vespa.hosted.controller.api.integration.cost.Backend; +import com.yahoo.vespa.hosted.controller.api.integration.cost.Cost; +import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException; + +import java.util.List; + +/** + * @author mpolden + */ +public class CostMock implements Cost { + + private final Backend backend; + + public CostMock(Backend backend) { + this.backend = backend; + } + + @Override + public List<ApplicationCost> getCPUAnalysis(int nofApplications) { + return null; + } + + @Override + public String getCsvForLocalAnalysis() { + return null; + } + + @Override + public List<ApplicationCost> getApplicationCost() { + return backend.getApplicationCost(); + } + + @Override + public ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId app) throws NotFoundCheckedException { + return backend.getApplicationCost(env, region, app); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java new file mode 100644 index 00000000000..c4ba5fa4fc5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java @@ -0,0 +1,41 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.cost; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; +import com.yahoo.vespa.hosted.controller.api.integration.cost.Backend; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class MockInsightBackend extends AbstractComponent implements Backend { + + private final Map<ApplicationId, ApplicationCost> applicationCost = new HashMap<>(); + + @Override + public List<ApplicationCost> getApplicationCost() { + return new ArrayList<>(applicationCost.values()); + } + + /** + * Get cost for a specific application in one zone or null if this application is not known. + * The zone information is ignored in the dummy backend. + */ + @Override + public ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId application) { + return applicationCost.get(application); + } + + public void setApplicationCost(ApplicationId application, ApplicationCost cost) { + applicationCost.put(application, cost); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java new file mode 100644 index 00000000000..aa115421f6a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -0,0 +1,113 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * A builder that builds application packages for testing purposes. + * + * @author mpolden + */ +public class ApplicationPackageBuilder { + + private String upgradePolicy = null; + private Environment environment = Environment.prod; + private final StringBuilder environmentBody = new StringBuilder(); + private final StringBuilder validationOverridesBody = new StringBuilder(); + + public ApplicationPackageBuilder upgradePolicy(String upgradePolicy) { + this.upgradePolicy = upgradePolicy; + return this; + } + + public ApplicationPackageBuilder environment(Environment environment) { + this.environment = environment; + return this; + } + + public ApplicationPackageBuilder region(String regionName) { + environmentBody.append(" <region active='true'>"); + environmentBody.append(regionName); + environmentBody.append("</region>\n"); + return this; + } + + public ApplicationPackageBuilder delay(Duration delay) { + environmentBody.append(" <delay seconds='"); + environmentBody.append(delay.getSeconds()); + environmentBody.append("'/>\n"); + return this; + } + + public ApplicationPackageBuilder allow(ValidationId validationId) { + validationOverridesBody.append(" <allow until='"); + validationOverridesBody.append(asIso8601String(Instant.now().plus(Duration.ofDays(29)))); + validationOverridesBody.append("'>"); + validationOverridesBody.append(validationId.value()); + validationOverridesBody.append("</allow>\n"); + return this; + } + + private byte[] deploymentSpec() { + StringBuilder xml = new StringBuilder("<deployment version='1.0'>\n"); + if (upgradePolicy != null) { + xml.append("<upgrade policy='"); + xml.append(upgradePolicy); + xml.append("'/>\n"); + } + xml.append(" <"); + xml.append(environment.value()); + xml.append(">\n"); + xml.append(environmentBody); + xml.append(" </"); + xml.append(environment.value()); + xml.append(">\n</deployment>"); + return xml.toString().getBytes(StandardCharsets.UTF_8); + } + + private byte[] validationOverrides() { + String xml = "<validation-overrides version='1.0'>\n" + + validationOverridesBody + + "</validation-overrides>\n"; + return xml.getBytes(StandardCharsets.UTF_8); + } + + public ApplicationPackage build() { + ByteArrayOutputStream zip = new ByteArrayOutputStream(); + ZipOutputStream out = new ZipOutputStream(zip); + try { + out.putNextEntry(new ZipEntry("deployment.xml")); + out.write(deploymentSpec()); + out.closeEntry(); + out.putNextEntry(new ZipEntry("validation-overrides.xml")); + out.write(validationOverrides()); + out.closeEntry(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + try { + out.close(); + } catch (IOException ignored) {} + } + return new ApplicationPackage(zip.toByteArray()); + } + + private static String asIso8601String(Instant instant) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE.withZone(ZoneId.systemDefault() ); + return formatter.format(instant); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java new file mode 100644 index 00000000000..32d1714ea52 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -0,0 +1,209 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.maintenance.FailureRedeployer; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class DeploymentTester { + + private ControllerTester tester = new ControllerTester(); + + private Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(2), + new JobControl(tester.curator())); + private FailureRedeployer failureRedeployer = new FailureRedeployer(tester.controller(), Duration.ofMinutes(2), + new JobControl(tester.curator())); + + public Upgrader upgrader() { return upgrader; } + public FailureRedeployer failureRedeployer() { return failureRedeployer; } + public Controller controller() { return tester.controller(); } + public ApplicationController applications() { return tester.controller().applications(); } + public BuildSystem buildSystem() { return tester.controller().applications().deploymentTrigger().buildSystem(); } + public DeploymentTrigger deploymentTrigger() { return tester.controller().applications().deploymentTrigger(); } + public ManualClock clock() { return tester.clock(); } + public ControllerTester controllerTester() { return tester; } + + public Application application(String name) { + return application(ApplicationId.from("tenant1", name, "default")); + } + + public Application application(ApplicationId application) { + return controller().applications().require(application); + } + + public Optional<Change.VersionChange> versionChange(ApplicationId application) { + return application(application).deploying() + .filter(c -> c instanceof Change.VersionChange) + .map(Change.VersionChange.class::cast); + } + + public ConfigServerClientMock configServerClientMock() { return tester.configServerClientMock(); } + + public void updateVersionStatus(Version currentVersion) { + controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion)); + } + + public void upgradeSystem(Version version) { + updateVersionStatus(version); + upgrader().maintain(); + } + + public Application createApplication(String applicationName, String tenantName, long projectId, Long propertyId) { + TenantId tenant = tester.createTenant(tenantName, UUID.randomUUID().toString(), propertyId); + return tester.createApplication(tenant, applicationName, "default", projectId); + } + + public void restartController() { tester.createNewController(); } + + /** Simulate the full lifecycle of an application deployment as declared in given application package */ + public Application createAndDeploy(String applicationName, int projectId, ApplicationPackage applicationPackage) { + tester.createTenant("tenant1", "domain1", 1L); + Application application = tester.createApplication(new TenantId("tenant1"), applicationName, "default", projectId); + deployCompletely(application, applicationPackage); + return applications().require(application.id()); + } + + /** Simulate the full lifecycle of an application deployment to prod.us-west-1 with the given upgrade policy */ + public Application createAndDeploy(String applicationName, int projectId, String upgradePolicy) { + return createAndDeploy(applicationName, projectId, applicationPackage(upgradePolicy)); + } + + /** Complete an ongoing deployment */ + public void deployCompletely(String applicationName) { + deployCompletely(applications().require(ApplicationId.from("tenant1", applicationName, "default")), + applicationPackage("default")); + } + + /** Deploy application completely using the given application package */ + public void deployCompletely(Application application, ApplicationPackage applicationPackage) { + notifyJobCompletion(JobType.component, application, true); + assertTrue(applications().require(application.id()).deploying().isPresent()); + completeDeployment(application, applicationPackage, Optional.empty()); + } + + private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) { + List<JobType> triggerOrder = JobType.triggerOrder(SystemName.main, applicationPackage.deploymentSpec()); + for (JobType job : triggerOrder) { + boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false); + deployAndNotify(job, application, applicationPackage, !failJob); + if (failJob) { + break; + } + } + if (failOnJob.isPresent()) { + assertTrue(applications().require(application.id()).deploying().isPresent()); + assertTrue(applications().require(application.id()).deploymentJobs().hasFailures()); + } else { + assertFalse(applications().require(application.id()).deploying().isPresent()); + } + } + + public void notifyJobCompletion(JobType jobType, Application application, boolean success) { + notifyJobCompletion(jobType, application, DeploymentJobs.JobError.from(success)); + } + + public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError) { + applications().notifyJobCompletion(jobReport(application, jobType, jobError)); + } + + public void completeUpgrade(Application application, Version version, String upgradePolicy) { + assertTrue(applications().require(application.id()).deploying().isPresent()); + assertEquals(new Change.VersionChange(version), applications().require(application.id()).deploying().get()); + completeDeployment(application, applicationPackage(upgradePolicy), Optional.empty()); + } + + public void completeUpgradeWithError(Application application, Version version, String upgradePolicy, JobType failOnJob) { + completeUpgradeWithError(application, version, applicationPackage(upgradePolicy), Optional.of(failOnJob)); + } + + public void completeUpgradeWithError(Application application, Version version, ApplicationPackage applicationPackage, JobType failOnJob) { + completeUpgradeWithError(application, version, applicationPackage, Optional.of(failOnJob)); + } + + private void completeUpgradeWithError(Application application, Version version, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) { + assertTrue(applications().require(application.id()).deploying().isPresent()); + assertEquals(new Change.VersionChange(version), applications().require(application.id()).deploying().get()); + completeDeployment(application, applicationPackage, failOnJob); + } + + public void deploy(JobType job, Application application, ApplicationPackage applicationPackage) { + deploy(job, application, applicationPackage, false); + } + + public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { + job.zone(SystemName.main).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); + } + + public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) { + assertScheduledJob(application, job); + if (success) { + deploy(job, application, applicationPackage); + } + notifyJobCompletion(job, application, success); + } + + private void assertScheduledJob(Application application, JobType jobType) { + Optional<BuildService.BuildJob> job = findJob(application, jobType); + assertTrue(String.format("Job %s is scheduled for %s", jobType, application), job.isPresent()); + buildSystem().removeJobs(application.id()); + assertEquals((long) application.deploymentJobs().projectId().get(), job.get().projectId()); + assertEquals(jobType.id(), job.get().jobName()); + } + + private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) { + for (BuildService.BuildJob job : buildSystem().jobs()) + if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id())) + return Optional.of(job); + return Optional.empty(); + } + + private DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { + return new DeploymentJobs.JobReport( + application.id(), + jobType, + application.deploymentJobs().projectId().get(), + 1L, + jobError, + false, + true + ); + } + + private static ApplicationPackage applicationPackage(String upgradePolicy) { + return new ApplicationPackageBuilder() + .upgradePolicy(upgradePolicy) + .environment(Environment.prod) + .region("us-west-1") + .build(); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java new file mode 100644 index 00000000000..ce06910240b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -0,0 +1,167 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + * @author mpolden + */ +public class DeploymentTriggerTest { + + @Test + public void testTriggerFailing() { + DeploymentTester tester = new DeploymentTester(); + Application app1 = tester.createAndDeploy("app1", 1, "default"); + + Version version = new Version(5, 2); + tester.deploymentTrigger().triggerChange(app1.id(), new Change.VersionChange(version)); + tester.completeUpgradeWithError(app1, version, "default", JobType.stagingTest); + assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size()); + + tester.buildSystem().takeJobsToRun(); + assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); + tester.clock().advance(Duration.ofHours(2)); + tester.deploymentTrigger().triggerFailing(app1.id()); + assertEquals("Retried job", 1, tester.buildSystem().jobs().size()); + assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); + + tester.buildSystem().takeJobsToRun(); + assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); + tester.clock().advance(Duration.ofHours(7)); + tester.deploymentTrigger().triggerFailing(app1.id()); + assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size()); + assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName()); + } + + @Test + public void deploymentSpecDecidesTriggerOrder() { + DeploymentTester tester = new DeploymentTester(); + BuildSystem buildSystem = tester.buildSystem(); + TenantId tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L); + Application application = tester.controllerTester().createApplication(tenant, "app1", "default", 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-central-1") + .region("us-west-1") + .build(); + + // Component job finishes + tester.notifyJobCompletion(JobType.component, application, true); + + // Application is deployed to all test environments and declared zones + tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true); + tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true); + tester.deployAndNotify(JobType.productionCorpUsEast1, application, applicationPackage, true); + tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true); + tester.deployAndNotify(JobType.productionUsWest1, application, applicationPackage, true); + assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); + } + + @Test + public void deploymentsSpecWithDelays() { + DeploymentTester tester = new DeploymentTester(); + BuildSystem buildSystem = tester.buildSystem(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .delay(Duration.ofSeconds(30)) + .region("us-west-1") + .delay(Duration.ofMinutes(1)) + .delay(Duration.ofMinutes(2)) // Multiple delays are summed up + .region("us-central-1") + .delay(Duration.ofMinutes(10)) // Delays after last region are valid, but have no effect + .build(); + + // Component job finishes + tester.notifyJobCompletion(JobType.component, application, true); + + // Test jobs pass + tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true); + tester.clock().advance(Duration.ofSeconds(1)); // Make staging test sort as the last successful job + tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true); + assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); + + // 30 seconds pass, us-west-1 is triggered + tester.clock().advance(Duration.ofSeconds(30)); + tester.deploymentTrigger().triggerDelayed(); + + // Consume us-west-1 job without reporting completion + assertEquals(1, buildSystem.jobs().size()); + assertEquals(JobType.productionUsWest1.id(), buildSystem.jobs().get(0).jobName()); + buildSystem.takeJobsToRun(); + + // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress + tester.clock().advance(Duration.ofMinutes(3)); + tester.deploymentTrigger().triggerDelayed(); + assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); + + // us-west-1 completes + tester.deploy(JobType.productionUsWest1, application, applicationPackage); + tester.notifyJobCompletion(JobType.productionUsWest1, application, true); + + // Delayed trigger does nothing as not enough time has passed after us-west-1 completion + tester.deploymentTrigger().triggerDelayed(); + assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); + + // 3 minutes pass, us-central-1 is triggered + tester.clock().advance(Duration.ofMinutes(3)); + tester.deploymentTrigger().triggerDelayed(); + tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true); + assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); + + // Delayed trigger job runs again, with nothing to trigger + tester.clock().advance(Duration.ofMinutes(10)); + tester.deploymentTrigger().triggerDelayed(); + assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); + } + + + @Test + public void testSuccessfulDeploymentApplicationPackageChanged() { + DeploymentTester tester = new DeploymentTester(); + BuildSystem buildSystem = tester.buildSystem(); + TenantId tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L); + Application application = tester.controllerTester().createApplication(tenant, "app1", "default", 1L); + ApplicationPackage previousApplicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-central-1") + .region("us-west-1") + .build(); + ApplicationPackage newApplicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-central-1") + .region("us-west-1") + .region("ap-northeast-1") + .build(); + + // Component job finishes + tester.notifyJobCompletion(JobType.component, application, true); + + // Application is deployed to all test environments and declared zones + tester.deployAndNotify(JobType.systemTest, application, newApplicationPackage, true); + tester.deploy(JobType.stagingTest, application, previousApplicationPackage, true); + tester.deployAndNotify(JobType.stagingTest, application, newApplicationPackage, true); + tester.deployAndNotify(JobType.productionCorpUsEast1, application, newApplicationPackage, true); + tester.deployAndNotify(JobType.productionUsCentral1, application, newApplicationPackage, true); + tester.deployAndNotify(JobType.productionUsWest1, application, newApplicationPackage, true); + tester.deployAndNotify(JobType.productionApNortheast1, application, newApplicationPackage, true); + assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java new file mode 100644 index 00000000000..6346d1cbdb6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java @@ -0,0 +1,179 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; + +import java.time.Duration; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.QUEUED; +import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.RUNNING; + +/** + * Simulates polling of build jobs from the controller and triggering and execution of + * these in Screwdriver. + * + * @author jvenstad + */ +public class MockBuildService implements BuildService { + + private final ControllerTester tester; + private final MockTimeline timeline; + private final Map<String, Job> jobs; + private final Map<String, JobStatus> jobStatuses; + private Version version; + + public MockBuildService(ControllerTester tester, MockTimeline timeline) { + this.tester = tester; + this.timeline = timeline; + jobs = new HashMap<>(); + jobStatuses = new HashMap<>(); + version = new Version(6, 86); + } + + /** Simulates the triggering of a Screwdriver job, where jobs are queued if already running. */ + @Override + public boolean trigger(BuildJob buildJob) { + String key = buildJob.toString(); + System.err.println(timeline.now() + ": Asked to trigger " + key); + + if ( ! jobStatuses.containsKey(key)) + startJob(key); + else + jobStatuses.put(key, QUEUED); + + return true; + } + + /** Simulates the internal triggering of Screwdriver, where only one instance is run at a time. */ + private void startJob(String key) { + jobStatuses.put(key, RUNNING); + Job job = jobs.get(key); + if (job == null) + return; + + timeline.in(job.duration, () -> { + job.outcome(); + if (jobStatuses.get(key) == QUEUED) + startJob(key); + else + jobStatuses.remove(key); + }); + System.err.println(timeline.now() + ": Triggered " + key + "; it will finish at " + timeline.now().plus(job.duration)); + } + + public void incrementVersion() { + version = new Version(version.getMajor(), version.getMinor() + 1); + } + + public Version version() { return version; } + + /** Add @job to the set of @Job objects we have information about. */ + private void add(Job job) { + jobs.put(job.buildJob().toString(), job); + } + + /** Add @project to the set of @Project objects we have information about. */ + private void add(Project project) { + project.jobs.values().forEach(this::add); + } + + /** Make a @Project with the given settings, modify it if desired, and @add() it its jobs to the pool of known ones. */ + public Project project(ApplicationId applicationId, Long projectId, Duration duration, Supplier<Boolean> success) { + return new Project(applicationId, projectId, duration, success); + } + + + /** Convenience creator for many jobs, belonging to the same project. Jobs can be modified independently after creation. */ + class Project { + + private final ApplicationId applicationId; + private final Long projectId; + private final Duration duration; + private final Supplier<Boolean> success; + private final Map<JobType, Job> jobs; + + private Project(ApplicationId applicationId, Long projectId, Duration duration, Supplier<Boolean> success) { + this.applicationId = applicationId; + this.projectId = projectId; + this.duration = duration; + this.success = success; + + jobs = new EnumMap<>(JobType.class); + + for (JobType jobType : JobType.values()) + jobs.put(jobType, new Job(applicationId, projectId, jobType, duration, success)); + } + + /** Set @duration for @jobType of this @Project. */ + public Project set(Duration duration, JobType jobType) { + jobs.compute(jobType, (type, job) -> new Job(applicationId, projectId, jobType, duration, job.success)); + return this; + } + + /** Set @success for @jobType of this @Project. */ + public Project set(Supplier<Boolean> success, JobType jobType) { + jobs.compute(jobType, (type, job) -> new Job(applicationId, projectId, jobType, job.duration, success)); + return this; + } + + /** Add the @Job objects of this @Project to the pool of known jobs for this @MockBuildService. */ + public void add() { + MockBuildService.this.add(this); + } + + } + + + /** Representation of a simulated job -- most noteworthy is the @outcome(), which is used to simulate a job completing. */ + private class Job { + + private final ApplicationId applicationId; + private final Long projectId; + private final JobType jobType; + private final Duration duration; + private final Supplier<Boolean> success; + + private Job(ApplicationId applicationId, Long projectId, JobType jobType, Duration duration, Supplier<Boolean> success) { + this.applicationId = applicationId; + this.projectId = projectId; + this.jobType = jobType; + this.duration = duration; + this.success = success; + } + + private void outcome() { + Boolean success = this.success.get(); + System.err.println(timeline.now() + ": Job " + projectId + ":" + jobType + " reports " + success); + if (success != null) + tester.controller().applications().notifyJobCompletion( + new DeploymentJobs.JobReport( + applicationId, + jobType, + projectId, + 1L, + JobError.from(success), + false, + false + )); + } + + private BuildJob buildJob() { return new BuildJob(projectId, jobType.id()); } + + } + + enum JobStatus { + QUEUED, + RUNNING + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java new file mode 100644 index 00000000000..878c25bf6bd --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java @@ -0,0 +1,106 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.test.ManualClock; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.PriorityQueue; + +/** + * @author jvenstad + */ +public class MockTimeline { + + private final ManualClock clock; + private final PriorityQueue<Event> events; + + public MockTimeline(ManualClock clock) { + this.events = new PriorityQueue<>(); + this.clock = clock; + } + + /** Make @event happen at time @at, as measured by the internal clock. */ + public void at(Instant at, Runnable event) { + if (at.isBefore(now())) + throw new IllegalArgumentException("The flow of time runs only one way, my friend."); + events.add(new Event(at, event)); + } + + /** Make @event happen in @in time, as measured by the internal clock. */ + public void in(Duration in, Runnable event) { + at(now().plus(in), event); + } + + /** Make @event happen every @period time, starting @offset time from @now(), as measured by the internal clock. */ + public void every(Duration period, Duration offset, Runnable event) { + in(offset, () -> { + every(period, event); + event.run(); + }); + } + + /** Make @event happen every @period time, starting @period time from @now(), as measured by the internal clock. */ + public void every(Duration period, Runnable event) { + every(period, period, event); + } + + /** Returns the current time, as measured by the internal clock. */ + public Instant now() { + return clock.instant(); + } + + /** Returns whether there are more events in the timeline, or not. */ + public boolean hasNext() { + return ! events.isEmpty(); + } + + /** Advance time to the next event, let it happen, and return the time of this event. */ + public Instant next() { + Event event = events.poll(); + clock.advance(Duration.ofMillis(now().until(event.at(), ChronoUnit.MILLIS))); + event.happen(); + return event.at(); + } + + /** Advance the time until @until, letting all events from now to then happen. */ + public void advance(Instant until) { + at(until, () -> {}); + while (next() != until); + } + + /** Advance the time by @duration, letting all events from now to then happen. */ + public void advance(Duration duration) { + advance(now().plus(duration)); + } + + /** Let the timeline unfold! Careful about those @every-s, though... */ + public void unfold() { + while (hasNext()) + next(); + } + + + private static class Event implements Comparable<Event> { + + private final Instant at; + private final Runnable event; + + private Event(Instant at, Runnable event) { + this.at = at; + this.event = event; + } + + public Instant at() { return at; } + public void happen() { event.run(); } + + + @Override + public int compareTo(Event other) { + return at().compareTo(other.at()); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java new file mode 100644 index 00000000000..779af370ff4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +@RunWith(Parameterized.class) +public class PolledBuildSystemTest { + + @Parameterized.Parameters(name = "jobType={0}") + public static Iterable<? extends Object> capacityConstrainedJobs() { + return Arrays.asList(JobType.systemTest, JobType.stagingTest); + } + + private final JobType jobType; + + public PolledBuildSystemTest(JobType jobType) { + this.jobType = jobType; + } + + @Test + public void throttle_capacity_constrained_jobs() { + ControllerTester tester = new ControllerTester(); + BuildSystem buildSystem = new PolledBuildSystem(tester.controller(), new MockCuratorDb()); + + long fooProjectId = 1; + long barProjectId = 2; + ApplicationId foo = tester.createAndDeploy("tenant1", "domain1", "app1", + Environment.prod, fooProjectId).id(); + ApplicationId bar = tester.createAndDeploy("tenant2", "domain2", "app2", + Environment.prod, barProjectId).id(); + + // Trigger jobs in capacity constrained environment + buildSystem.addJob(foo, jobType, false); + buildSystem.addJob(bar, jobType, false); + + // Capacity constrained jobs are returned one a at a time + List<BuildJob> nextJobs = buildSystem.takeJobsToRun(); + assertEquals(1, nextJobs.size()); + assertEquals(fooProjectId, nextJobs.get(0).projectId()); + + nextJobs = buildSystem.takeJobsToRun(); + assertEquals(1, nextJobs.size()); + assertEquals(barProjectId, nextJobs.get(0).projectId()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java new file mode 100644 index 00000000000..360fd8616d3 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.integration; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; + +/** + * @author bratseth + */ +public class MockMetricsService implements com.yahoo.vespa.hosted.controller.api.integration.MetricsService { + + @Override + public ApplicationMetrics getApplicationMetrics(ApplicationId application) { + return new ApplicationMetrics(0.5, 0.7); + } + + @Override + public DeploymentMetrics getDeploymentMetrics(ApplicationId application, Zone zone) { + return new DeploymentMetrics(1, 2, 3, 4, 5); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java new file mode 100644 index 00000000000..4c53a6d37e4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -0,0 +1,46 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; + +import java.io.IOException; +import java.time.Duration; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class DeploymentExpirerTest { + + @Test + public void testDeploymentExpiry() throws IOException, InterruptedException { + ControllerTester tester = new ControllerTester(); + tester.getZoneRegistryMock().setDeploymentTimeToLive(new Zone(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14)); + DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10), + tester.clock(), new JobControl(new MockCuratorDb())); + ApplicationId devApp = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); + ApplicationId prodApp = tester.createAndDeploy("tenant2", "domain2", "app2", Environment.prod, 456).id(); + + assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size()); + assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); + + // Not expired at first + expirer.maintain(); + assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size()); + assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); + + // The dev application is removed + tester.clock().advance(Duration.ofDays(15)); + expirer.maintain(); + assertEquals(0, tester.controller().applications().get(devApp).get().deployments().size()); + assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java new file mode 100644 index 00000000000..f5a76f6446c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -0,0 +1,280 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.Contacts.UserContact; +import com.yahoo.vespa.hosted.controller.api.integration.Issues; +import com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.ContactsMock; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.PropertiesMock; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; +import org.junit.Test; + +import java.time.Clock; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.admin; +import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.engineeringOwner; +import static com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo.Status.done; +import static com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo.Status.toDo; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxFailureAge; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxInactivityAge; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.terminalUser; +import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.vespaOps; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author jvenstad + */ +public class DeploymentIssueReporterTest { + + private final static ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + + private DeploymentTester tester; + private DeploymentIssueReporter reporter; + private ContactsMock contacts; + private PropertiesMock properties; + private MockIssues issues; + + @Before + public void setup() { + tester = new DeploymentTester(); + contacts = new ContactsMock(); + properties = new PropertiesMock(); + issues = new MockIssues(tester.clock()); + reporter = new DeploymentIssueReporter(tester.controller(), contacts, properties, issues, Duration.ofMinutes(5), + new JobControl(new MockCuratorDb())); + } + + private List<IssueInfo> openIssuesFor(Application application) { + return issues.fetchSimilarTo(reporter.deploymentIssueFrom(tester.controller().applications().require(application.id()))); + } + + @Test + public void testDeploymentFailureReporting() { + // All applications deploy from unique SD projects. + Long projectId1 = 10L; + Long projectId2 = 20L; + Long projectId3 = 30L; + + // Only the first two have propertyIds set now. + Long propertyId1 = 1L; + Long propertyId2 = 2L; + + // Create and deploy one application for each of three tenants. + Application app1 = tester.createApplication("application1", "tenant1", projectId1, propertyId1); + Application app2 = tester.createApplication("application2", "tenant2", projectId2, propertyId2); + Application app3 = tester.createApplication("application3", "tenant3", projectId3, null); + + // And then we need lots of successful applications, so we won't assume we just have a faulty Vespa out. + for (long i = 4; i <= 10; i++) { + Application app = tester.createApplication("application" + i, "tenant" + i, 10 * i, i); + tester.notifyJobCompletion(component, app, true); + tester.deployAndNotify(systemTest, app, applicationPackage, true); + tester.deployAndNotify(stagingTest, app, applicationPackage, true); + tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true); + } + + // Both the first tenants belong to the same JIRA queue. (Not sure if this is possible, but let's test it anyway. + String jiraQueue = "PROJECT"; + properties.addClassification(propertyId1, jiraQueue); + properties.addClassification(propertyId1, jiraQueue); + + // Only tenant1 has contacts listed in opsDb. + UserContact + alice = new UserContact("alice", "Alice", admin), + bob = new UserContact("bob", "Robert", engineeringOwner); + contacts.addContact(propertyId1, Arrays.asList(alice, bob)); + + // end of setup. + + // NOTE: All maintenance should be idempotent within a small enough time interval, so maintain is called twice in succession throughout. + + // app1 and app3 has one failure each. + tester.notifyJobCompletion(component, app1, true); + tester.deployAndNotify(systemTest, app1, applicationPackage, true); + tester.deployAndNotify(stagingTest, app1, applicationPackage, false); + + tester.notifyJobCompletion(component, app2, true); + tester.deployAndNotify(systemTest, app2, applicationPackage, true); + tester.deployAndNotify(stagingTest, app2, applicationPackage, true); + + tester.notifyJobCompletion(component, app3, true); + tester.deployAndNotify(systemTest, app3, applicationPackage, true); + tester.deployAndNotify(stagingTest, app3, applicationPackage, true); + tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false); + + reporter.maintain(); + reporter.maintain(); + assertEquals("No deployments are detected as failing for a long time initially.", 0, issues.issues.size()); + + + // Advance to where deployment issues should be detected. + tester.clock().advance(maxFailureAge.plus(Duration.ofDays(1))); + + reporter.maintain(); + reporter.maintain(); + assertEquals("One issue is produced for app1.", 1, openIssuesFor(app1).size()); + assertEquals("No issues are produced for app2.", 0, openIssuesFor(app2).size()); + assertEquals("One issue is produced for app3.", 1, openIssuesFor(app3).size()); + assertTrue("The issue for app1 is stored in their JIRA queue.", openIssuesFor(app1).get(0).key().startsWith(jiraQueue)); + assertTrue("The issue for an application without propertyId is addressed to vespaOps.", openIssuesFor(app3).get(0).key().startsWith(vespaOps.queue())); + + + // Verify idempotency of filing. + reporter.maintain(); + reporter.maintain(); + assertEquals("No issues are re-filed when still open.", 2, issues.issues.size()); + + + // tenant3 closes their issue prematurely; see that we get a new filing. + issues.complete(openIssuesFor(app3).get(0).id()); + assertEquals("The issue is removed (test of the tester, really...).", 0, openIssuesFor(app3).size()); + + reporter.maintain(); + reporter.maintain(); + assertTrue("Issue is re-produced for app3, addressed correctly.", openIssuesFor(app3).get(0).key().startsWith(vespaOps.queue())); + + + // Some time passes; tenant1 leaves her issue unattended, while tenant3 starts work and updates the issue. + // app2 also has an intermittent failure; see that we detect this as a Vespa problem, and file an issue to ourselves. + tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, false); + tester.clock().advance(maxInactivityAge.plus(maxFailureAge)); + issues.comment(openIssuesFor(app3).get(0).id(), "We are trying to fix it!"); + + reporter.maintain(); + reporter.maintain(); + assertEquals("The issue for app1 is escalated once.", alice.username(), openIssuesFor(app1).get(0).assignee().get()); + + + reporter.maintain(); + reporter.maintain(); + assertEquals("We get an issue to vespaOps when more than 20% of applications have old failures.", 1, + issues.fetchSimilarTo(reporter.manyFailingDeploymentsIssueFrom(Arrays.asList( + tester.controller().applications().get(app1.id()).get(), + tester.controller().applications().get(app2.id()).get(), + tester.controller().applications().get(app3.id()).get()))).size()); + assertEquals("No issue is filed for app2 while Vespa is considered broken.", 0, openIssuesFor(app2).size()); + + + // app3 fixes its problem, but the ticket is left open; see the resolved ticket is not escalated when another escalation period has passed. + tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, true); + tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, true); + tester.clock().advance(maxInactivityAge.plus(Duration.ofDays(1))); + + reporter.maintain(); + reporter.maintain(); + assertEquals("The issue for app1 is escalated once more.", bob.username(), openIssuesFor(app1).get(0).assignee().get()); + assertEquals("The issue for app3 is still unassigned.", Optional.empty(), openIssuesFor(app3).get(0).assignee()); + + + // app1 still does nothing with their issue; see the terminal user gets it in the end. + // app3 now has a new failure past max failure age; see that a new issue is filed. + tester.notifyJobCompletion(component, app3, true); + tester.deployAndNotify(systemTest, app3, applicationPackage, true); + tester.deployAndNotify(stagingTest, app3, applicationPackage, true); + tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false); + tester.clock().advance(maxInactivityAge.plus(maxFailureAge)); + + reporter.maintain(); + reporter.maintain(); + assertEquals("The issue for app1 is escalated to the terminal user.", terminalUser.username(), openIssuesFor(app1).get(0).assignee().get()); + assertEquals("A new issue is filed for app3.", 2, openIssuesFor(app3).size()); + } + + class MockIssues implements Issues { + + final Map<String, Issue> issues = new HashMap<>(); + final Map<String, IssueInfo> metas = new HashMap<>(); + final Map<String, Long> counters = new HashMap<>(); + Clock clock; + + MockIssues(Clock clock) { this.clock = clock; } + + public void addWatcher(String jiraIssueId, String watcher) { + touch(jiraIssueId); + } + + public void reassign(String jiraIssueId, String assignee) { + metas.compute(jiraIssueId, (__, jiraIssueMeta) -> + new IssueInfo( + jiraIssueId, + jiraIssueMeta.key(), + clock.instant(), + Optional.of(assignee), + jiraIssueMeta.status())); + } + + public void comment(String jiraIssueId, String comment) { + touch(jiraIssueId); + } + + public void update(String jiraIssueId, String description) { + issues.compute(jiraIssueId, (__, issue) -> + new Issue(issue.summary(), description, issue.classification().orElse(null))); + } + + public String file(Issue issue) { + String jiraIssueId = (issues.size() + 1L) + ""; + Long counter = counters.merge(issue.classification().get().queue(), 0L, (old, __) -> old + 1); + String jiraIssueKey = issue.classification().get().queue() + '-' + counter; + issues.put(jiraIssueId, issue); + metas.put(jiraIssueId, new IssueInfo(jiraIssueId, jiraIssueKey, clock.instant(), null, toDo)); + return jiraIssueId; + } + + public IssueInfo fetch(String jiraIssueId) { + return metas.get(jiraIssueId); + } + + public List<IssueInfo> fetchSimilarTo(Issue issue) { + return issues.entrySet().stream() + .filter(entry -> entry.getValue().summary().equals(issue.summary())) + .map(Map.Entry::getKey) + .map(metas::get) + .filter(meta -> meta.status() != done) + .collect(Collectors.toList()); + } + + private void complete(String jiraIssueId) { + metas.compute(jiraIssueId, (__, jiraIssueMeta) -> + new IssueInfo( + jiraIssueId, + jiraIssueMeta.key(), + clock.instant(), + jiraIssueMeta.assignee(), + done)); + } + + private void touch(String jiraIssueId) { + metas.compute(jiraIssueId, (__, jiraIssueMeta) -> + new IssueInfo( + jiraIssueId, + jiraIssueMeta.key(), + clock.instant(), + jiraIssueMeta.assignee(), + jiraIssueMeta.status())); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java new file mode 100644 index 00000000000..cde511a9076 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -0,0 +1,169 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class FailureRedeployerTest { + + @Test + public void testRetryingFailedJobsDuringDeployment() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .environment(Environment.prod) + .region("us-east-3") + .build(); + Version version = Version.fromString("5.0"); + tester.updateVersionStatus(version); + + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + + // New version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Test environments pass + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + + // Production job fails and is retried + tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress + tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false); + assertEquals("Production job is retried", 1, tester.buildSystem().jobs().size()); + assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); + + // Another version is released, which cancels any pending upgrades to lower versions + version = Version.fromString("5.2"); + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals("Application starts upgrading to new version", 1, tester.buildSystem().jobs().size()); + assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); + + // Failure redeployer does not retry failing job for prod.us-east-3 as there's an ongoing deployment + tester.clock().advance(Duration.ofMinutes(1)); + tester.failureRedeployer().maintain(); + assertFalse("Job is not retried", tester.buildSystem().jobs().stream() + .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id()))); + + // Test environments pass + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + + // Production job fails again and exhausts all immediate retries + tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, false); + assertTrue("Retries exhausted", tester.buildSystem().jobs().isEmpty()); + assertTrue("Failure is recorded", tester.application(app.id()).deploymentJobs().hasFailures()); + + // Failure redeployer retries job + tester.clock().advance(Duration.ofMinutes(5)); + tester.failureRedeployer().maintain(); + assertEquals("Job is retried", 1, tester.buildSystem().jobs().size()); + + // Production job finally succeeds + tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); + assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures()); + } + + @Test + public void testRetriesDeploymentWithStuckJobs() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .environment(Environment.prod) + .region("us-east-3") + .build(); + + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + + // staging-test starts, but does not complete + assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + tester.failureRedeployer().maintain(); + assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); + + // Just over 12 hours pass, deployment is retried from beginning + tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); + tester.failureRedeployer().maintain(); + assertEquals(DeploymentJobs.JobType.component.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + + // Ensure that system-test is trigered after component. Triggering component records a new change, but in this + // case there's already a change in progress which we want to discard and start over + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + assertEquals(DeploymentJobs.JobType.systemTest.id(), tester.buildSystem().jobs().get(0).jobName()); + } + + @Test + public void testRetriesJobsFailingForCurrentChange() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .environment(Environment.prod) + .region("us-east-3") + .build(); + Version version = Version.fromString("5.0"); + tester.updateVersionStatus(version); + + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + + // New version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); + + // system-test fails and exhausts all immediate retries + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, false); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, app, false); + assertTrue("Retries exhausted", tester.buildSystem().jobs().isEmpty()); + + // Another version is released + version = Version.fromString("5.2"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); + + // Consume system-test job for 5.2 + tester.buildSystem().takeJobsToRun(); + + // Failure re-deployer does not retry failing system-test job as it failed for an older change + tester.clock().advance(Duration.ofMinutes(5)); + tester.failureRedeployer().maintain(); + assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java new file mode 100644 index 00000000000..44d8adf1d15 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java @@ -0,0 +1,93 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class JobControlTest { + + @Test + public void testJobControl() { + JobControl jobControl = new JobControl(new MockCuratorDb()); + + assertTrue(jobControl.jobs().isEmpty()); + + String job1 = "Job1"; + String job2 = "Job2"; + + jobControl.started(job1); + jobControl.started(job2); + assertEquals(2, jobControl.jobs().size()); + assertTrue(jobControl.jobs().contains(job1)); + assertTrue(jobControl.jobs().contains(job2)); + + assertTrue(jobControl.isActive(job1)); + assertTrue(jobControl.isActive(job2)); + + jobControl.setActive(job1, false); + assertFalse(jobControl.isActive(job1)); + assertTrue(jobControl.isActive(job2)); + + jobControl.setActive(job2, false); + assertFalse(jobControl.isActive(job1)); + assertFalse(jobControl.isActive(job2)); + + jobControl.setActive(job1, true); + assertTrue(jobControl.isActive(job1)); + assertFalse(jobControl.isActive(job2)); + + jobControl.setActive(job2, true); + assertTrue(jobControl.isActive(job1)); + assertTrue(jobControl.isActive(job2)); + } + + @Test + public void testJobControlMayDeactivateJobs() { + JobControl jobControl = new JobControl(new MockCuratorDb()); + + ControllerTester tester = new ControllerTester(); + MockMaintainer mockMaintainer = new MockMaintainer(tester.controller(), jobControl); + + assertTrue(jobControl.jobs().contains("MockMaintainer")); + + assertEquals(0, mockMaintainer.maintenanceInvocations); + + mockMaintainer.run(); + assertEquals(1, mockMaintainer.maintenanceInvocations); + + jobControl.setActive("MockMaintainer", false); + mockMaintainer.run(); + assertEquals(1, mockMaintainer.maintenanceInvocations); + + jobControl.setActive("MockMaintainer", true); + mockMaintainer.run(); + assertEquals(2, mockMaintainer.maintenanceInvocations); + } + + private static class MockMaintainer extends Maintainer { + + int maintenanceInvocations = 0; + + public MockMaintainer(Controller controller, JobControl jobControl) { + super(controller, Duration.ofHours(1), jobControl); + } + + @Override + protected void maintain() { + maintenanceInvocations++; + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java new file mode 100644 index 00000000000..a832a591217 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -0,0 +1,132 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.MetricsMock; +import com.yahoo.vespa.hosted.controller.MetricsMock.MapContext; +import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; +import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Map; + +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static org.fest.assertions.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyListOf; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; + +/** + * @author mortent + */ +public class MetricsReporterTest { + + @Test + public void test_chef_metrics() throws IOException { + ControllerTester tester = new ControllerTester(); + MetricsMock metricsMock = new MetricsMock(); + MetricsReporter metricsReporter = setupMetricsReporter(tester.controller(), metricsMock, SystemName.cd); + metricsReporter.maintain(); + assertEquals(2, metricsMock.getMetrics().size()); + + Map<MapContext, Map<String, Number>> metrics = metricsMock.getMetricsFilteredByHost("fake-node.test"); + assertEquals(1, metrics.size()); + Map.Entry<MapContext, Map<String, Number>> metricEntry = metrics.entrySet().iterator().next(); + MapContext metricContext = metricEntry.getKey(); + assertDimension(metricContext, "tenantName", "ciintegrationtests"); + assertDimension(metricContext, "app", "restart.default"); + assertDimension(metricContext, "zone", "prod.cd-us-east-1"); + assertThat(metricEntry.getValue().get(MetricsReporter.convergeMetric).longValue()).isEqualTo(727); + } + + @Test + public void test_deployment_metrics() throws IOException { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .build(); + MetricsMock metricsMock = new MetricsMock(); + MetricsReporter metricsReporter = setupMetricsReporter(tester.controller(), metricsMock, SystemName.cd); + + metricsReporter.maintain(); + assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); + + // Deploy 3 apps successfully + Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); + Application app2 = tester.createApplication("app2", "tenant1", 2, 22L); + Application app3 = tester.createApplication("app3", "tenant1", 3, 33L); + Application app4 = tester.createApplication("app4", "tenant1", 4, 44L); + tester.deployCompletely(app1, applicationPackage); + tester.deployCompletely(app2, applicationPackage); + tester.deployCompletely(app3, applicationPackage); + + metricsReporter.maintain(); + assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); + + // 1 app fails system-test + tester.notifyJobCompletion(component, app4, true); + tester.deployAndNotify(systemTest, app4, applicationPackage, false); + + metricsReporter.maintain(); + assertEquals(25.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); + } + + @Test + public void it_omits_zone_when_unknown() throws IOException { + ControllerTester tester = new ControllerTester(); + String hostname = "fake-node2.test"; + MapContext metricContext = getMetricsForHost(tester.controller(), hostname); + assertThat(metricContext.getDimensions().get("zone")).isNull(); + } + + private void assertDimension(MapContext metricContext, String dimensionName, String expectedValue) { + assertThat(metricContext.getDimensions().get(dimensionName)).isNotNull().isEqualTo(expectedValue); + } + + private MetricsReporter setupMetricsReporter(Controller controller, MetricsMock metricsMock, SystemName system) throws IOException { + Chef client = Mockito.mock(Chef.class); + PartialNodeResult result = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .readValue(getClass().getClassLoader().getResource("chef_output.json"), PartialNodeResult.class); + when(client.partialSearchNodes(anyString(), anyListOf(AttributeMapping.class))).thenReturn(result); + + Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault()); + + return new MetricsReporter(controller, metricsMock, client, clock, + new JobControl(new MockCuratorDb()), system); + } + + private MapContext getMetricsForHost(Controller controller, String hostname) throws IOException { + MetricsMock metricsMock = new MetricsMock(); + MetricsReporter metricsReporter = setupMetricsReporter(controller, metricsMock, SystemName.main); + metricsReporter.maintain(); + + assertThat(metricsMock.getMetrics()).isNotEmpty(); + + Map<MapContext, Map<String, Number>> metrics = metricsMock.getMetricsFilteredByHost(hostname); + assertThat(metrics).hasSize(1); + Map.Entry<MapContext, Map<String, Number>> metricEntry = metrics.entrySet().iterator().next(); + return metricEntry.getKey(); + } + +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java new file mode 100644 index 00000000000..78b4f7f895f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class OutstandingChangeDeployerTest { + + @Test + public void testChangeDeployer() { + DeploymentTester tester = new DeploymentTester(); + OutstandingChangeDeployer deployer = new OutstandingChangeDeployer(tester.controller(), Duration.ofMinutes(10), + new JobControl(new MockCuratorDb())); + + tester.createAndDeploy("app1", 11, "default"); + tester.createAndDeploy("app2", 22, "default"); + + Version version = new Version(5, 2); + tester.deploymentTrigger().triggerChange(tester.application("app1").id(), new Change.VersionChange(version)); + + assertEquals(new Change.VersionChange(version), tester.application("app1").deploying().get()); + assertFalse(tester.application("app1").hasOutstandingChange()); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, tester.application("app1"), true); + assertTrue(tester.application("app1").hasOutstandingChange()); + assertEquals(1, tester.buildSystem().jobs().size()); + + deployer.maintain(); + assertEquals("No effect as job is in progress", 1, tester.buildSystem().jobs().size()); + + tester.deployCompletely("app1"); + assertEquals("Upgrade done", 0, tester.buildSystem().jobs().size()); + + deployer.maintain(); + List<BuildService.BuildJob> jobs = tester.buildSystem().jobs(); + assertEquals(1, jobs.size()); + assertEquals(11, jobs.get(0).projectId()); + assertEquals(DeploymentJobs.JobType.systemTest.id(), jobs.get(0).jobName()); + assertFalse(tester.application("app1").hasOutstandingChange()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java new file mode 100644 index 00000000000..e5afcec87ad --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -0,0 +1,304 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class UpgraderTest { + + @Test + public void testUpgrading() { + // --- Setup + DeploymentTester tester = new DeploymentTester(); + tester.upgrader().maintain(); + assertEquals("No system version: Nothing to do", 0, tester.buildSystem().jobs().size()); + + Version version = Version.fromString("5.0"); // (lower than the hardcoded version in the config server client) + tester.updateVersionStatus(version); + + tester.upgrader().maintain(); + assertEquals("No applications: Nothing to do", 0, tester.buildSystem().jobs().size()); + + // Setup applications + Application canary0 = tester.createAndDeploy("canary0", 0, "canary"); + Application canary1 = tester.createAndDeploy("canary1", 1, "canary"); + Application default0 = tester.createAndDeploy("default0", 2, "default"); + Application default1 = tester.createAndDeploy("default1", 3, "default"); + Application default2 = tester.createAndDeploy("default2", 4, "default"); + Application conservative0 = tester.createAndDeploy("conservative0", 5, "conservative"); + + tester.upgrader().maintain(); + assertEquals("All already on the right version: Nothing to do", 0, tester.buildSystem().jobs().size()); + + // --- A new version is released - everything goes smoothly + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); + tester.completeUpgrade(canary0, version, "canary"); + assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get()); + + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals("One canary pending; nothing else", 1, tester.buildSystem().jobs().size()); + + tester.completeUpgrade(canary1, version, "canary"); + + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Canaries done: Should upgrade defaults", 3, tester.buildSystem().jobs().size()); + + tester.completeUpgrade(default0, version, "default"); + tester.completeUpgrade(default1, version, "default"); + tester.completeUpgrade(default2, version, "default"); + + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Normals done: Should upgrade conservatives", 1, tester.buildSystem().jobs().size()); + tester.completeUpgrade(conservative0, version, "conservative"); + + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size()); + + // --- A new version is released - which fails a Canary + version = Version.fromString("5.2"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); + tester.completeUpgradeWithError(canary0, version, "canary", DeploymentJobs.JobType.stagingTest); + assertEquals("Other Canary was cancelled", 2, tester.buildSystem().jobs().size()); + + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Version broken, but Canaries should keep trying", 2, tester.buildSystem().jobs().size()); + + // --- A new version is released - which repairs the Canary app and fails a default + version = Version.fromString("5.3"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); + tester.completeUpgrade(canary0, version, "canary"); + assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get()); + + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals("One canary pending; nothing else", 1, tester.buildSystem().jobs().size()); + + tester.completeUpgrade(canary1, version, "canary"); + + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + + assertEquals("Canaries done: Should upgrade defaults", 3, tester.buildSystem().jobs().size()); + + tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgrade(default1, version, "default"); + tester.completeUpgrade(default2, version, "default"); + + tester.updateVersionStatus(version); + assertEquals("Not enough evidence to mark this neither broken nor high", + VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Upgrade with error should retry", 1, tester.buildSystem().jobs().size()); + + // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold + // Deploy application change + tester.deployCompletely("default0"); + // Complete upgrade + tester.upgrader().maintain(); + tester.completeUpgrade(default0, version, "default"); + + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Normals done: Should upgrade conservatives", 1, tester.buildSystem().jobs().size()); + tester.completeUpgrade(conservative0, version, "conservative"); + + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size()); + } + + @Test + public void testUpgradingToVersionWhichBreaksSomeNonCanaries() { + // --- Setup + DeploymentTester tester = new DeploymentTester(); + tester.upgrader().maintain(); + assertEquals("No system version: Nothing to do", 0, tester.buildSystem().jobs().size()); + + Version version = Version.fromString("5.0"); // (lower than the hardcoded version in the config server client) + tester.updateVersionStatus(version); + + tester.upgrader().maintain(); + assertEquals("No applications: Nothing to do", 0, tester.buildSystem().jobs().size()); + + // Setup applications + Application canary0 = tester.createAndDeploy("canary0", 0, "canary"); + Application canary1 = tester.createAndDeploy("canary1", 1, "canary"); + Application default0 = tester.createAndDeploy("default0", 2, "default"); + Application default1 = tester.createAndDeploy("default1", 3, "default"); + Application default2 = tester.createAndDeploy("default2", 4, "default"); + Application default3 = tester.createAndDeploy("default3", 5, "default"); + Application default4 = tester.createAndDeploy("default4", 6, "default"); + Application default5 = tester.createAndDeploy("default5", 7, "default"); + Application default6 = tester.createAndDeploy("default6", 8, "default"); + Application default7 = tester.createAndDeploy("default7", 9, "default"); + Application default8 = tester.createAndDeploy("default8", 10, "default"); + Application default9 = tester.createAndDeploy("default9", 11, "default"); + + tester.upgrader().maintain(); + assertEquals("All already on the right version: Nothing to do", 0, tester.buildSystem().jobs().size()); + + // --- A new version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); + tester.completeUpgrade(canary0, version, "canary"); + assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get()); + + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals("One canary pending; nothing else", 1, tester.buildSystem().jobs().size()); + + tester.completeUpgrade(canary1, version, "canary"); + + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Canaries done: Should upgrade defaults", 10, tester.buildSystem().jobs().size()); + + tester.completeUpgrade(default0, version, "default"); + tester.completeUpgradeWithError(default1, version, "default", DeploymentJobs.JobType.systemTest); + tester.completeUpgradeWithError(default2, version, "default", DeploymentJobs.JobType.systemTest); + tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.systemTest); + tester.completeUpgradeWithError(default4, version, "default", DeploymentJobs.JobType.systemTest); + + // > 40% and at least 4 failed - version is broken + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + assertEquals("Upgrades are cancelled", 0, tester.buildSystem().jobs().size()); + } + + @Test + public void testDeploymentAlreadyInProgressForUpgrade() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .environment(Environment.prod) + .region("us-east-3") + .build(); + Version version = Version.fromString("5.0"); + tester.updateVersionStatus(version); + + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true); + + tester.upgrader().maintain(); + assertEquals("Application is on expected version: Nothing to do", 0, + tester.buildSystem().jobs().size()); + + // New version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // system-test completes successfully + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + + // staging-test fails multiple times, exhausts retries and failure is recorded + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, false); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, app, false); + assertTrue("Retries exhausted", tester.buildSystem().jobs().isEmpty()); + assertTrue("Failure is recorded", tester.application(app.id()).deploymentJobs().hasFailures()); + assertTrue("Application has pending change", tester.application(app.id()).deploying().isPresent()); + + // New version is released + version = Version.fromString("5.2"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Upgrade is scheduled. system-tests starts, but does not complete + tester.upgrader().maintain(); + assertTrue("Application still has failures", tester.application(app.id()).deploymentJobs().hasFailures()); + assertEquals(1, tester.buildSystem().jobs().size()); + tester.buildSystem().takeJobsToRun(); + + // Upgrader runs again, nothing happens as there's already a job in progress for this change + tester.upgrader().maintain(); + assertTrue("No more jobs triggered at this time", tester.buildSystem().jobs().isEmpty()); + } + + // TODO: Remove when corp-prod special casing is no longer needed + @Test + public void upgradesCanariesToControllerVersion() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + + Version version = Version.fromString("5.0"); // Lower version than controller (6.10) + tester.updateVersionStatus(version); + + // Application is on 5.0 + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true); + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true); + + // Canary in prod.corp-us-east-1 is upgraded to controller version + tester.upgrader().maintain(); + assertEquals("Upgrade started", 1, tester.buildSystem().jobs().size()); + assertEquals(Vtag.currentVersion, ((Change.VersionChange) tester.application(app.id()).deploying().get()).version()); + tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); + tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true); + + // System is upgraded to newer version, no upgrade triggered for canary as version is lower than controller + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + tester.upgrader().maintain(); + assertTrue("No more jobs triggered", tester.buildSystem().jobs().isEmpty()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java new file mode 100644 index 00000000000..a7cecda3695 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import org.junit.Test; + +import java.time.Duration; +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class VersionStatusUpdaterTest { + + /** Test that this job updates the status. Test of the content of the update is in VersionStatusTest */ + @Test + public void testVersionUpdating() { + ControllerTester tester = new ControllerTester(); + tester.controller().updateVersionStatus(new VersionStatus(Collections.emptyList())); + assertFalse(tester.controller().versionStatus().systemVersion().isPresent()); + + VersionStatusUpdater updater = new VersionStatusUpdater(tester.controller(), Duration.ofMinutes(3), + new JobControl(new MockCuratorDb())); + updater.maintain(); + assertTrue(tester.controller().versionStatus().systemVersion().isPresent()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java new file mode 100644 index 00000000000..645e38d0f2d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -0,0 +1,196 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author bratseth + */ +public class ApplicationSerializerTest { + + private static final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + + private static final Zone zone1 = new Zone(Environment.from("prod"), RegionName.from("us-west-1")); + private static final Zone zone2 = new Zone(Environment.from("prod"), RegionName.from("us-east-3")); + + @Test + public void testSerialization() { + ControllerTester tester = new ControllerTester(); + DeploymentSpec deploymentSpec = DeploymentSpec.fromXml("<deployment version='1.0'>" + + " <staging/>" + + "</deployment>"); + ValidationOverrides validationOverrides = ValidationOverrides.fromXml("<validation-overrides version='1.0'>" + + " <allow until='2017-06-15'>deployment-removal</allow>" + + "</validation-overrides>"); + + List<Deployment> deployments = new ArrayList<>(); + ApplicationRevision revision1 = ApplicationRevision.from("appHash1"); + ApplicationRevision revision2 = ApplicationRevision.from("appHash2", new SourceRevision("repo1", "branch1", "commit1")); + deployments.add(new Deployment(zone1, revision1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); + deployments.add(new Deployment(zone2, revision2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5))); + + Optional<Long> projectId = Optional.of(123L); + List<JobStatus> statusList = new ArrayList<>(); + + statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest) + .withTriggering(Version.fromString("5.6.7"), Optional.empty(), Instant.ofEpochMilli(7)) + .withCompletion(Optional.empty(), Instant.ofEpochMilli(8), tester.controller())); + statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest) + .withTriggering(Version.fromString("5.6.6"), Optional.empty(), Instant.ofEpochMilli(5)) + .withCompletion(Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller())); + + DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty(), false); + + Application original = new Application(ApplicationId.from("t1", "a1", "i1"), + deploymentSpec, + validationOverrides, + deployments, deploymentJobs, + Optional.of(new Change.VersionChange(Version.fromString("6.7"))), + true); + + Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); + + assertEquals(original.id(), serialized.id()); + + assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm()); + assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm()); + + assertEquals(2, serialized.deployments().size()); + assertEquals(original.deployments().get(zone1).revision(), serialized.deployments().get(zone1).revision()); + assertEquals(original.deployments().get(zone2).revision(), serialized.deployments().get(zone2).revision()); + assertEquals(original.deployments().get(zone1).version(), serialized.deployments().get(zone1).version()); + assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version()); + assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at()); + assertEquals(original.deployments().get(zone2).at(), serialized.deployments().get(zone2).at()); + + assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId()); + assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size()); + assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest), + serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest)); + assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), + serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest)); + assertEquals(original.deploymentJobs().failingSince(), serialized.deploymentJobs().failingSince()); + assertEquals(original.deploymentJobs().isSelfTriggering(), serialized.deploymentJobs().isSelfTriggering()); + + assertEquals(original.hasOutstandingChange(), serialized.hasOutstandingChange()); + + assertEquals(original.deploying(), serialized.deploying()); + + { // test more deployment serialization cases + Application original2 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1")))); + Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2)); + assertEquals(original2.deploying(), serialized2.deploying()); + assertEquals(((Change.ApplicationChange)serialized2.deploying().get()).revision().get().source(), + ((Change.ApplicationChange)original2.deploying().get()).revision().get().source()); + + Application original3 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1", + new SourceRevision("a", "b", "c"))))); + Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3)); + assertEquals(original3.deploying(), serialized2.deploying()); + assertEquals(((Change.ApplicationChange)serialized3.deploying().get()).revision().get().source(), + ((Change.ApplicationChange)original3.deploying().get()).revision().get().source()); + + Application original4 = original.withDeploying(Optional.empty()); + Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4)); + assertEquals(original4.deploying(), serialized4.deploying()); + } + } + + @Test + public void testLegacySerialization() throws IOException { + Application applicationWithSuccessfulJob = applicationSerializer.fromSlime(applicationSlime(false)); + assertFalse("No job error for successful job", applicationWithSuccessfulJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().isPresent()); + + Application applicationWithFailingJob = applicationSerializer.fromSlime(applicationSlime(true)); + assertEquals(JobError.unknown, applicationWithFailingJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().get()); + } + + // TODO: Remove after Aug 2017 + @Test + public void serializeWithRemovedZone() throws Exception { + String json = "{\n" + + " \"id\": \"t1:a1:i1\",\n" + + " \"deploymentSpecField\": \"<deployment version='1.0'/>\",\n" + + " \"deploymentJobs\": {\n" + + " \"projectId\": 123,\n" + + " \"jobStatus\": [\n" + + " {\n" + + " \"jobType\": \"system-test\",\n" + + " \"version\": \"5.6.7\",\n" + + " \"completionTime\": 7,\n" + + " \"lastTriggered\": 8\n" + + " },\n" + + " {\n" + + " \"jobType\": \"production-ap-aue-1\",\n" + + " \"version\": \"5.6.7\",\n" + + " \"completionTime\": 7,\n" + + " \"lastTriggered\": 8\n" + + " },\n" + + " {\n" + + " \"jobType\": \"staging-test\",\n" + + " \"version\": \"5.6.7\",\n" + + " \"completionTime\": 7,\n" + + " \"lastTriggered\": 8\n" + + " }\n" + + " ],\n" + + " \"selfTriggering\": false\n" + + " }\n" + + "}\n"; + Application app = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(json.getBytes(StandardCharsets.UTF_8))); + assertEquals(2, app.deploymentJobs().jobStatus().size()); + } + + private Slime applicationSlime(boolean error) { + return SlimeUtils.jsonToSlime(applicationJson(error).getBytes(StandardCharsets.UTF_8)); + } + + private String applicationJson(boolean error) { + return + "{\n" + + " \"id\": \"t1:a1:i1\",\n" + + " \"deploymentSpecField\": \"<deployment version='1.0'/>\",\n" + + " \"deploymentJobs\": {\n" + + " \"projectId\": 123,\n" + + " \"jobStatus\": [\n" + + " {\n" + + " \"jobType\": \"system-test\",\n" + + " \"version\": \"5.6.7\",\n" + + " \"completionTime\": 7,\n" + + (error ? " \"jobError\": \"" + JobError.unknown + "\",\n" : "") + + " \"lastTriggered\": 8\n" + + " }\n" + + " ],\n" + + " \"selfTriggering\": false\n" + + " }\n" + + "}\n"; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java new file mode 100644 index 00000000000..348b9c92614 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -0,0 +1,101 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.TestIdentities; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; +import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; +import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; + +import java.io.File; +import java.io.IOException; +import java.util.Optional; + +/** + * Provides testing of controller functionality accessed through the container + * + * @author bratseth + */ +public class ContainerControllerTester { + + private final ContainerTester containerTester; + private final Controller controller; + + public ContainerControllerTester(JDisc container, String responseFilePath) { + containerTester = new ContainerTester(container, responseFilePath); + controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller"); + } + + public Controller controller() { return controller; } + + /** Returns the wrapped generic container tester */ + public ContainerTester containerTester() { return containerTester; } + + public Application createApplication() { + AthensDomain domain1 = addTenantAthensDomain("domain1", "mytenant"); + controller.tenants().addTenant(Tenant.createAthensTenant(new TenantId("tenant1"), domain1, + new Property("property1"), + Optional.of(new PropertyId("1234"))), + Optional.of(TestIdentities.userNToken)); + ApplicationId app = ApplicationId.from("tenant1", "application1", "default"); + return controller.applications().createApplication(app, Optional.of(TestIdentities.userNToken)); + } + + public Application deploy(Application application, ApplicationPackage applicationPackage, Zone zone, long projectId) { + ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(projectId)); + GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); + controller.applications().deployApplication(application.id(), + zone, + applicationPackage, + new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), Optional.empty(), false, false)); + return application; + } + + public void notifyJobCompletion(ApplicationId applicationId, long projectId, boolean success, DeploymentJobs.JobType job) { + controller().applications().notifyJobCompletion(new DeploymentJobs.JobReport(applicationId, job, projectId, 1L, + success ? Optional.empty() : Optional.of(DeploymentJobs.JobError.unknown), + false, false)); + } + + public AthensDomain addTenantAthensDomain(String domainName, String userName) { + Athens athens = (AthensMock) containerTester.container().components().getComponent( + "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock" + ); + ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory(); + AthensDomain athensDomain = new AthensDomain(domainName); + AthensDbMock.Domain domain = new AthensDbMock.Domain(athensDomain); + domain.markAsVespaTenant(); + domain.admin(new AthensPrincipal(new AthensDomain("domain"), new UserId(userName))); + mock.getSetup().addDomain(domain); + return athensDomain; + } + + // ---- Delegators: + + public void assertResponse(Request request, File expectedResponse) throws IOException { + containerTester.assertResponse(request, expectedResponse); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java new file mode 100644 index 00000000000..7a9e74a3c27 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -0,0 +1,136 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.collections.Pair; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * Provides testing of JSON container responses + * + * @author bratseth + */ +public class ContainerTester { + + private final JDisc container; + private final String responseFilePath; + + public ContainerTester(JDisc container, String responseFilePath) { + this.container = container; + this.responseFilePath = responseFilePath; + } + + public JDisc container() { return container; } + + public void updateSystemVersion() { + Controller controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller"); + controller.updateVersionStatus(VersionStatus.compute(controller)); + } + + public void assertResponse(Request request, File responseFile) throws IOException { + assertResponse(request, responseFile, 200); + } + + public void assertResponse(Request request, File responseFile, int expectedStatusCode) throws IOException { + String expectedResponse = IOUtils.readFile(new File(responseFilePath + responseFile.toString())); + expectedResponse = include(expectedResponse); + Response response = container.handleRequest(request); + Slime expectedSlime = SlimeUtils.jsonToSlime(expectedResponse.getBytes(StandardCharsets.UTF_8)); + Set<String> fieldsToCensor = fieldsToCensor(null, expectedSlime.get(), new HashSet<>()); + Slime responseSlime = SlimeUtils.jsonToSlime(response.getBody()); + List<Pair<String,String>> replaceStrings = new ArrayList<>(); + buildReplaceStrings(null, responseSlime.get(), fieldsToCensor, replaceStrings); + + String body = response.getBodyAsString(); + assertEquals("Status code. Response body was: " + body, expectedStatusCode, response.getStatus()); + assertEquals(responseFile.toString(), new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), + replace(new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8), replaceStrings)); + } + + public void assertResponse(Request request, String expectedResponse) throws IOException { + assertResponse(request, expectedResponse, 200); + } + + public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException { + Response response = container.handleRequest(request); + assertEquals("Status code", expectedStatusCode, response.getStatus()); + assertEquals(expectedResponse, response.getBodyAsString()); + } + + private Set<String> fieldsToCensor(String fieldNameOrNull, Inspector value, Set<String> fieldsToCensor) { + switch (value.type()) { + case ARRAY: value.traverse((ArrayTraverser)(int index, Inspector element) -> fieldsToCensor(null, element, fieldsToCensor)); break; + case OBJECT: value.traverse((String fieldName, Inspector fieldValue) -> fieldsToCensor(fieldName, fieldValue, fieldsToCensor)); break; + case STRING: if (fieldNameOrNull != null && "(ignore)".equals(value.asString())) fieldsToCensor.add(fieldNameOrNull); break; + } + return fieldsToCensor; + } + + private void buildReplaceStrings(String fieldNameOrNull, Inspector value, Set<String> fieldsToCensor, + List<Pair<String,String>> replaceStrings) { + switch (value.type()) { + case ARRAY: value.traverse((ArrayTraverser)(int index, Inspector element) -> buildReplaceStrings(null, element, fieldsToCensor, replaceStrings)); break; + case OBJECT: value.traverse((String fieldName, Inspector fieldValue) -> buildReplaceStrings(fieldName, fieldValue, fieldsToCensor, replaceStrings)); break; + default: replaceString(fieldNameOrNull, value, fieldsToCensor, replaceStrings); + } + } + + private void replaceString(String fieldName, Inspector fieldValue, + Set<String> fieldsToCensor, List<Pair<String,String>> replaceStrings) { + if (fieldName == null) return; + if ( ! fieldsToCensor.contains(fieldName)) return; + + String fromString; + if ( fieldValue.type().equals(Type.STRING)) + fromString = "\"" + fieldName + "\":\"" + fieldValue.asString() + "\""; + else if ( fieldValue.type().equals(Type.LONG)) + fromString = "\"" + fieldName + "\":" + fieldValue.asLong(); + else + throw new IllegalArgumentException("Can only censor strings and longs"); + String toString = "\"" + fieldName + "\":\"(ignore)\""; + replaceStrings.add(new Pair<>(fromString, toString)); + } + + private String replace(String json, List<Pair<String,String>> replaceStrings) { + for (Pair<String,String> replaceString : replaceStrings) + json = json.replace(replaceString.getFirst(), replaceString.getSecond()); + return json; + } + + /** Replaces @include(localFile) with the content of the file */ + private String include(String response) throws IOException { + // Please don't look at this code + int includeIndex = response.indexOf("@include("); + if (includeIndex < 0) return response; + String prefix = response.substring(0, includeIndex); + String rest = response.substring(includeIndex + "@include(".length()); + int filenameEnd = rest.indexOf(")"); + String includeFileName = rest.substring(0, filenameEnd); + String includedContent = IOUtils.readFile(new File(responseFilePath + includeFileName)); + includedContent = include(includedContent); + String postFix = rest.substring(filenameEnd + 1); + postFix = include(postFix); + return prefix + includedContent + postFix; + } + +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java new file mode 100644 index 00000000000..8b2595c6254 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -0,0 +1,85 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Superclass of REST API tests which needs to set up a functional container instance. + * + * This is a test superclass, not a tester because we need the start and stop methods. + * + * DO NOT ADD ANYTHING HERE: If you need additional fields and methods, create a tester + * which gets the container instance at construction time (in the test method) instead. + * + * @author bratseth + */ +public class ControllerContainerTest { + + protected JDisc container; + @Before + public void startContainer() { container = JDisc.fromServicesXml(controllerServicesXml, Networking.disable); } + @After + public void stopContainer() { container.close(); } + + private final String controllerServicesXml = + "<jdisc version='1.0'>" + + " <config name='vespa.hosted.zone.config.zone'>" + + " <system>main</system>" + + " </config>" + + " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.jira.JiraMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.ContactsMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingIssues'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.PropertiesMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerClientMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.ZoneRegistryMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.Controller'/>" + + " <component id='com.yahoo.vespa.hosted.controller.cost.MockInsightBackend'/>" + + " <component id='com.yahoo.vespa.hosted.controller.cost.CostMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.integration.MockMetricsService'/>" + + " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>" + + " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>" + + " <component id='com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb'/>" + + " <component id='com.yahoo.vespa.hosted.controller.restapi.application.MockAuthorizer'/>" + + " <component id='com.yahoo.vespa.hosted.controller.routing.MockRoutingGenerator'/>" + + " <component id='com.yahoo.vespa.hosted.rotation.MemoryRotationRepository'/>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.RootHandler'>" + + " <binding>http://*/</binding>" + + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>" + + " <binding>http://*/application/v4/*</binding>" + + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>" + + " <binding>http://*/deployment/v1/*</binding>" + + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>" + + " <binding>http://*/controller/v1/*</binding>" + + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.screwdriver.ScrewdriverApiHandler'>" + + " <binding>http://*/screwdriver/v1/*</binding>" + + " </handler>" + + "</jdisc>"; + + protected void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { + Response response = container.handleRequest(request); + // Compare both status and message at once for easier diagnosis + assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, + "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java new file mode 100644 index 00000000000..9534dd65d7f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.application.container.handler.Request; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * @author bratseth + */ +public class RootHandlerTest extends ControllerContainerTest { + + @Test + public void testRootRequest() throws IOException { + ContainerTester tester = new ContainerTester(container, + "src/test/java/com/yahoo/vespa/hosted/controller/restapi/"); + tester.assertResponse(new Request("http://localhost:8080/"), + new File("root-response.json"), 200); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java new file mode 100644 index 00000000000..97e1bac35c8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -0,0 +1,598 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; +import com.yahoo.vespa.hosted.controller.api.integration.cost.ClusterCost; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.cost.MockInsightBackend; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * @author bratseth + */ +public class ApplicationApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; + private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + private static final String athensUserDomain = "domain1"; + private static final String athensScrewdriverDomain = "screwdriver-domain"; + + @Test + public void testApplicationApi() throws IOException { + ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); + ContainerTester tester = controllerTester.containerTester(); + tester.updateSystemVersion(); + + addTenantAthensDomain(athensUserDomain, "mytenant"); // (Necessary but not provided in this API) + + // GET API root + tester.assertResponse(request("/application/v4/", "", Request.Method.GET), + new File("root.json")); + // GET athens domains + tester.assertResponse(request("/application/v4/athensDomain/", "", Request.Method.GET), + new File("athensDomain-list.json")); + // GET OpsDB properties + tester.assertResponse(request("/application/v4/property/", "", Request.Method.GET), + new File("property-list.json")); + // GET cookie freshness + tester.assertResponse(request("/application/v4/cookiefreshness/", "", Request.Method.GET), + new File("cookiefreshness.json")); + // POST (add) a tenant without property ID + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST), + new File("tenant-without-applications.json")); + // PUT (modify) a tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.PUT), + new File("tenant-without-applications.json")); + // GET the authenticated user (with associated tenants) + tester.assertResponse(request("/application/v4/user", "", Request.Method.GET), + new File("user.json")); + // GET all tenants + tester.assertResponse(request("/application/v4/tenant/", "", Request.Method.GET), + new File("tenant-list.json")); + // POST (create) an application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.POST), + new File("application-reference.json")); + // GET a tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.GET), + new File("tenant-with-application.json")); + // GET tenant applications + tester.assertResponse(request("/application/v4/tenant/tenant1/application/", "", Request.Method.GET), + new File("application-list.json")); + // POST triggering of a full deployment to an application (if version is omitted, current system version is used) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), + new File("application-deployment.json")); + + // POST (deploy) an application to a zone - manual user deployment + HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", + entity, + Request.Method.POST, + athensUserDomain, "mytenant"), + new File("deploy-result.json")); + + // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. + ApplicationId id = ApplicationId.from("tenant1", "application1", "default"); + long screwdriverProjectId = 123; + + addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // (Necessary but not provided in this API) + + // ... systemtest + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/test-region/instance/default/", + createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), + Request.Method.POST, + athensScrewdriverDomain, "screwdriveruser1"), + new File("deploy-result.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/test-region/instance/default", + "", + Request.Method.DELETE), + "Deactivated tenant/tenant1/application/application1/environment/test/region/test-region/instance/default"); + controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.systemTest); // Called through the separate screwdriver/v1 API + + // ... staging + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/staging-region/instance/default/", + createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), + Request.Method.POST, + athensScrewdriverDomain, "screwdriveruser1"), + new File("deploy-result.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/staging-region/instance/default", + "", + Request.Method.DELETE), + "Deactivated tenant/tenant1/application/application1/environment/staging/region/staging-region/instance/default"); + controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.stagingTest); + + // ... prod zone + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", + createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), + Request.Method.POST, + athensScrewdriverDomain, "screwdriveruser1"), + new File("deploy-result.json")); + controllerTester.notifyJobCompletion(id, screwdriverProjectId, false, DeploymentJobs.JobType.productionCorpUsEast1); + + // GET tenant screwdriver projects + tester.assertResponse(request("/application/v4/tenant-pipeline/", "", Request.Method.GET), + new File("tenant-pipelines.json")); + // GET tenant application deployments + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), + new File("application.json")); + // GET an application deployment + addMockObservedApplicationCost("tenant1", "application1", "default"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET), + new File("deployment.json")); + // POST a 'restart application' command + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", + "", + Request.Method.POST), + "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // POST a 'restart application' command with a host filter (other filters not supported yet) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", + "", + Request.Method.POST), + "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // POST a 'log' command + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/log", + "", + Request.Method.POST), + new File("log-response.json")); // Proxied to config server, not sure about the expected return format + // GET (wait for) convergence + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", "", Request.Method.GET), + new File("convergence.json")); + // GET services + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", "", Request.Method.GET), + new File("services.json")); + // GET service + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "", Request.Method.GET), + new File("service.json")); + // DELETE (deactivate) a deployment - dev + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", + "", + Request.Method.DELETE), + "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); + // DELETE (deactivate) a deployment - prod + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", + "", + Request.Method.DELETE), + "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); + // DELETE an application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.DELETE), + ""); + // DELETE a tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + new File("tenant-without-applications.json")); + + // PUT (create) the authenticated user + tester.assertResponse(request("/application/v4/user?user=newuser&domain=by", + new byte[0], + Request.Method.PUT, + athensUserDomain, "newuser", "application/json"), + new File("create-user-response.json")); + // OPTIONS return 200 OK + tester.assertResponse(request("/application/v4/", "", Request.Method.OPTIONS), + ""); + + // Add another Athens domain, so we can try to create more tenants + addTenantAthensDomain("domain2", "mytenant"); // New domain to test tenant w/property ID + // POST (add) a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", + "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}", + Request.Method.POST), + new File("tenant-without-applications-with-id.json")); + // PUT (modify) a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", + "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}", + Request.Method.PUT), + new File("tenant-without-applications-with-id.json")); + // GET a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", "", Request.Method.GET), + new File("tenant-without-applications-with-id.json")); + + // Test legacy OpsDB tenants + // POST (add) an OpsDB tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant3", + "{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}", + Request.Method.POST), + new File("opsdb-tenant-with-id-without-applications.json")); + // PUT (modify) the OpsDB tenant to set another property + tester.assertResponse(request("/application/v4/tenant/tenant3", + "{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}", + Request.Method.PUT), + new File("opsdb-tenant-with-new-id-without-applications.json")); + + // GET global rotation status + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", "", Request.Method.GET), + new File("global-rotation.json")); + + // GET global rotation override status + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", "", Request.Method.GET), + new File("global-rotation-get.json")); + + // SET global rotation override status + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.PUT), + new File("global-rotation-put.json")); + + // DELETE global rotation override status + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.DELETE), + new File("global-rotation-delete.json")); + + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", "", Request.Method.POST), + "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), + "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); + } + + @Test + public void testErrorResponses() throws IOException, URISyntaxException { + ContainerTester tester = new ContainerTester(container, responseFiles); + tester.updateSystemVersion(); + addTenantAthensDomain("domain1", "mytenant"); + + // PUT (update) non-existing tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.PUT), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", + 404); + + // GET non-existing tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", + "", + Request.Method.GET), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", + 404); + + // GET non-existing application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.GET), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}", + 404); + + // GET non-existing deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east/instance/default", + "", + Request.Method.GET), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}", + 404); + + // POST (add) a tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST), + new File("tenant-without-applications.json")); + + // POST (add) another tenant under the same domain + tester.assertResponse(request("/application/v4/tenant/tenant2", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create tenant 'tenant2': The Athens domain 'domain1' is already connected to tenant 'tenant1'\"}", + 400); + + // Add the same tenant again + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", + 400); + + // POST (create) an (empty) application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.POST), + new File("application-reference.json")); + + // Create the same application again + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.POST), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"An application with id 'tenant1.application1' already exists\"}", + 400); + + ConfigServerClientMock configServer = (ConfigServerClientMock)container.components().getComponent("com.yahoo.vespa.hosted.controller.ConfigServerClientMock"); + configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE, null)); + + // POST (deploy) an application with an invalid application package + HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", + entity, + Request.Method.POST, + athensUserDomain, "mytenant"), + new File("deploy-failure.json"), 400); + + // POST (deploy) an application without available capacity + configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", + entity, + Request.Method.POST, + athensUserDomain, "mytenant"), + new File("deploy-out-of-capacity.json"), 400); + + // DELETE tenant which has an application + tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not delete tenant 'tenant1': This tenant has active applications\"}", + 400); + + // DELETE application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.DELETE), + ""); + // DELETE application again - should produce 404 + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", + 404); + // DELETE tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + new File("tenant-without-applications.json")); + // DELETE tenant again - should produce 404 + tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete tenant 'tenant1': Tenant not found\"}", + 404); + + // Promote application chef env for nonexistent tenant/application + tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Unable to promote Chef environments for application\"}", + 500); + } + + @Test + public void testAuthorization() throws IOException, URISyntaxException { + ContainerTester tester = new ContainerTester(container, responseFiles); + String authorizedUser = "mytenant"; + String unauthorizedUser = "othertenant"; + + // Mutation without an authorized user is disallowed + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST, + "domain1", null), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"User is not authenticated\"}", + 403); + + // ... but read methods are allowed + tester.assertResponse(request("/application/v4/tenant/", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.GET, + "domain1", null), + "[]", + 200); + + addTenantAthensDomain("domain1", "mytenant"); + + // Creating a tenant for an Athens domain the user is not admin for is disallowed + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST, + "domain1", unauthorizedUser), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"The user 'othertenant' is not admin in Athens domain 'domain1'\"}", + 403); + + // (Create it with the right tenant id) + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST, + "domain1", authorizedUser), + new File("tenant-without-applications.json"), + 200); + + // Creating an application for an Athens domain the user is not admin for is disallowed + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.POST, + "domain1", unauthorizedUser), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", + 403); + + // (Create it with the right tenant id) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.POST, + "domain1", authorizedUser), + new File("application-reference.json"), + 200); + + // Deploy to an authorized zone by a user tenant is disallowed + HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", + entity, + Request.Method.POST, + athensUserDomain, "mytenant"), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Principal 'mytenant' is not a screwdriver principal, and does not have deploy access to application 'tenant1.application1'\"}", + 403); + + // Deleting an application for an Athens domain the user is not admin for is disallowed + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.DELETE, + "domain1", unauthorizedUser), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", + 403); + + // (Deleting it with the right tenant id) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.DELETE, + "domain1", authorizedUser), + "", + 200); + + // Updating a tenant for an Athens domain the user is not admin for is disallowed + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.PUT, + "domain1", unauthorizedUser), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", + 403); + + // Change Athens domain + addTenantAthensDomain("domain2", "mytenant"); + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain2\", \"property\":\"property1\"}", + Request.Method.PUT, + "domain1", authorizedUser), + "{\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}", + 200); + + // Deleting a tenant for an Athens domain the user is not admin for is disallowed + tester.assertResponse(request("/application/v4/tenant/tenant1", + "", + Request.Method.DELETE, + "domain1", unauthorizedUser), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", + 403); + } + + private HttpEntity createApplicationDeployData(ApplicationPackage applicationPackage, Optional<Long> screwdriverJobId) { + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("deployOptions", deployOptions(screwdriverJobId), ContentType.APPLICATION_JSON); + builder.addBinaryBody("applicationZip", applicationPackage.zippedContent()); + return builder.build(); + } + + private String deployOptions(Optional<Long> screwdriverJobId) { + if (screwdriverJobId.isPresent()) // deployment from screwdriver + return "{\"vespaVersion\":null," + + "\"ignoreValidationErrors\":false," + + "\"screwdriverBuildJob\":{\"screwdriverId\":\"" + screwdriverJobId.get() + "\"," + + "\"gitRevision\":{\"repository\":\"repository1\"," + + "\"branch\":\"master\"," + + "\"commit\":\"commit1\"" + + "}" + + "}" + + "}"; + else // This is ugly and evil, but tentatively replicates the existing behavor from the client on user deployments + return "{\"vespaVersion\":null," + + "\"ignoreValidationErrors\":false," + + "\"screwdriverBuildJob\":{\"screwdriverId\":null," + + "\"gitRevision\":{\"repository\":null," + + "\"branch\":null," + + "\"commit\":null" + + "}" + + "}" + + "}"; + + } + + /** Make a request with (athens) user domain1.mytenant1 */ + private Request request(String path, String data, Request.Method method) { + return request(path, data.getBytes(StandardCharsets.UTF_8), method, "domain1", "mytenant", "application/json"); + } + + private Request request(String path, String data, Request.Method method, String domain, String user) { + return request(path, data.getBytes(StandardCharsets.UTF_8), method, domain, user, "application/json"); + } + + private Request request(String path, byte[] data, Request.Method method, String domain, String user, String contentType) { + // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters + Request request = new Request("http://localhost:8080" + path + "?domain=" + domain + + (user != null ? "&user=" + user : ""), + data, method); + request.getHeaders().put("Content-Type", contentType); + return request; + } + + private Request request(String path, HttpEntity data, Request.Method method, String domain, String user) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + data.writeTo(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return request(path, out.toByteArray(), method, domain, user, data.getContentType().getValue()); + } + + /** + * In production this happens outside hosted Vespa, so there is no API for it and we need to reach down into the + * mock setup to replicate the action. + */ + private AthensDomain addTenantAthensDomain(String domainName, String userName) { + Athens athens = (AthensMock) container.components().getComponent( + "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock" + ); + ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory(); + AthensDomain athensDomain = new AthensDomain(domainName); + AthensDbMock.Domain domain = new AthensDbMock.Domain(athensDomain); + domain.markAsVespaTenant(); + domain.admin(new AthensPrincipal(new AthensDomain(athensUserDomain), new UserId(userName))); + mock.getSetup().addDomain(domain); + return athensDomain; + } + + /** + * In production this happens outside hosted Vespa, so there is no API for it and we need to reach down into the + * mock setup to replicate the action. + */ + private void addScrewdriverUserToDomain(String screwdriverUserId, String domainName) { + Athens athens = (AthensMock) container.components().getComponent( + "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock" + ); + ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory(); + AthensDbMock.Domain domain = mock.getSetup().domains.get(new AthensDomain(domainName)); + domain.admin(new AthensPrincipal(new AthensDomain(athensScrewdriverDomain), new UserId(screwdriverUserId))); + } + + private void addMockObservedApplicationCost(String tenant, String application, String instance) { + MockInsightBackend mock = (MockInsightBackend) container.components().getComponent("com.yahoo.vespa.hosted.controller.cost.MockInsightBackend"); + + ClusterCost cost = new ClusterCost(); + cost.setCount(2); + cost.setResource("cpu"); + cost.setUtilization(1.0f); + cost.setTco(25); + cost.setFlavor("flavor1"); + cost.setWaste(10); + cost.setType("content"); + List<String> hostnames = new ArrayList<>(); + hostnames.add("host1"); + hostnames.add("host2"); + cost.setHostnames(hostnames); + Map<String, ClusterCost> clusterCosts = new HashMap<>(); + clusterCosts.put("cluster1", cost); + + mock.setApplicationCost(new ApplicationId.Builder().tenant(tenant).applicationName(application).instanceName(instance).build(), + new ApplicationCost("prod.us-west-1", tenant, application + "." + instance, 37, 1.0f, 0.0f, clusterCosts)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java new file mode 100644 index 00000000000..16557157cf5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.TestIdentities; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; + +import javax.ws.rs.core.SecurityContext; +import java.security.Principal; +import java.util.Optional; + +/** + * This overrides methods in Authorizer which relies on properties set by jdisc HTTP filters. + * This is necessary because filters are not currently executed when executing requests with Application. + * + * @author bratseth + */ +@SuppressWarnings("unused") // injected +public class MockAuthorizer extends Authorizer { + + public MockAuthorizer(Controller controller, EntityService entityService) { + super(controller, entityService); + } + + /** Returns a principal given by the request parameters 'domain' and 'user' */ + @Override + public Optional<Principal> getPrincipalIfAny(HttpRequest request) { + if (request.getProperty("user") == null) return Optional.empty(); + return Optional.of(new AthensPrincipal(new AthensDomain(request.getProperty("domain")), + new UserId(request.getProperty("user")))); + } + + /** Returns the hardcoded NToken of {@link TestIdentities#userId} */ + @Override + public Optional<NToken> getNToken(HttpRequest request) { + return Optional.of(TestIdentities.userNToken); + } + + private static class MockPrincipal implements Principal { + + @Override + public String getName() { return TestIdentities.userId.id(); } + + } + + @Override + protected Optional<SecurityContext> securityContextOf(HttpRequest request) { + return getPrincipalIfAny(request).map(MockSecurityContext::new); + } + + private static final class MockSecurityContext implements SecurityContext { + + private final Principal principal; + + private MockSecurityContext(Principal principal) { + this.principal = principal; + } + + @Override + public Principal getUserPrincipal() { return principal; } + + @Override + public boolean isUserInRole(String role) { return false; } + + @Override + public boolean isSecure() { return true; } + + @Override + public String getAuthenticationScheme() { throw new UnsupportedOperationException(); } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java new file mode 100644 index 00000000000..1a623c4e3eb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java @@ -0,0 +1,91 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.google.inject.Key; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.Container; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.ResourceReference; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.jdisc.service.CurrentContainer; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class MultipartParserTest { + + @Test + public void multipartParserTest() throws URISyntaxException { + String data = + "Content-Type: multipart/form-data; boundary=AaB03x\r\n" + + "\r\n" + + "--AaB03x\r\n" + + "Content-Disposition: form-data; name=\"submit-name\"\r\n" + + "\r\n" + + "Larry\r\n" + + "--AaB03x\r\n" + + "Content-Disposition: form-data; name=\"submit-address\"\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + + "House 1\r\n" + + "--AaB03x\r\n" + + "Content-Disposition: form-data; name=\"files\"; filename=\"file1.txt\"\r\n" + + "Content-Type: text/plain\r\n" + + "\r\n" + + "... contents of file1.txt ...\r\n" + + "--AaB03x--\r\n"; + ByteArrayInputStream dataStream = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)); + HttpRequest request = HttpRequest.createRequest(new MockCurrentContainer(), + new URI("http://foo"), + com.yahoo.jdisc.http.HttpRequest.Method.POST, + dataStream); + request.getJDiscRequest().headers().put("Content-Type", "multipart/form-data; boundary=AaB03x"); + Map<String, byte[]> parts = new MultipartParser().parse(request); + assertEquals(3, parts.size()); + assertTrue(parts.containsKey("submit-name")); + assertTrue(parts.containsKey("submit-address")); + assertTrue(parts.containsKey("files")); + assertEquals("Larry", new String(parts.get("submit-name"), StandardCharsets.UTF_8)); + assertEquals("... contents of file1.txt ...", new String(parts.get("files"), StandardCharsets.UTF_8)); + } + + private static class MockCurrentContainer implements CurrentContainer { + + @Override + public Container newReference(URI uri) { return new MockContainer(); } + + } + + private static class MockContainer implements Container { + + @Override + public RequestHandler resolveHandler(Request request) { return null; } + + @Override + public <T> T getInstance(Key<T> key) { return null; } + + @Override + public <T> T getInstance(Class<T> aClass) { return null; } + + @Override + public ResourceReference refer() { return null; } + + @Override + public void release() { } + + @Override + public long currentTimeMillis() { return 0; } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java new file mode 100644 index 00000000000..7b1d8d17a5c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java @@ -0,0 +1,63 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.vespa.hosted.controller.restapi.Path; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class PathTest { + + @Test + public void testPath() { + assertFalse(new Path("").matches("/a/{foo}/bar/{b}"));; + assertFalse(new Path("///").matches("/a/{foo}/bar/{b}"));; + assertFalse(new Path("///foo").matches("/a/{foo}/bar/{b}"));; + assertFalse(new Path("///bar/").matches("/a/{foo}/bar/{b}"));; + Path path = new Path("/a/1/bar/fuz"); + assertTrue(path.matches("/a/{foo}/bar/{b}"));; + assertEquals("1", path.get("foo")); + assertEquals("fuz", path.get("b")); + } + + @Test + public void testPathWithRest() { + { + Path path = new Path("/a/1/bar/fuz/"); + assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); + assertEquals("1", path.get("foo")); + assertEquals("fuz", path.get("b")); + assertEquals("", path.getRest()); + } + + { + Path path = new Path("/a/1/bar/fuz/kanoo"); + assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); + assertEquals("1", path.get("foo")); + assertEquals("fuz", path.get("b")); + assertEquals("kanoo", path.getRest()); + } + + { + Path path = new Path("/a/1/bar/fuz/kanoo/trips"); + assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); + assertEquals("1", path.get("foo")); + assertEquals("fuz", path.get("b")); + assertEquals("kanoo/trips", path.getRest()); + } + + { + Path path = new Path("/a/1/bar/fuz/kanoo/trips/"); + assertTrue(path.matches("/a/{foo}/bar/{b}/{*}")); + assertEquals("1", path.get("foo")); + assertEquals("fuz", path.get("b")); + assertEquals("kanoo/trips/", path.getRest()); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java new file mode 100644 index 00000000000..6cf90905679 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -0,0 +1,95 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.io.IOUtils; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.ClusterView; +import com.yahoo.vespa.serviceview.bindings.ServiceView; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class ServiceApiResponseTest { + + private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; + + @Test + public void testServiceViewResponse() throws URISyntaxException, IOException { + ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.prod, RegionName.from("us-west-1")), + ApplicationId.from("tenant1", "application1", "default"), + Collections.singletonList(new URI("config-server1")), + new URI("http://server1:4080/request/path?foo=bar")); + ApplicationView applicationView = new ApplicationView(); + ClusterView clusterView = new ClusterView(); + clusterView.type = "container"; + clusterView.name = "cluster1"; + clusterView.url = "cluster-url"; + ServiceView serviceView = new ServiceView(); + serviceView.url = null; + serviceView.serviceType = "container"; + serviceView.serviceName = "service1"; + serviceView.configId = "configId1"; + serviceView.host = "host1"; + serviceView.legacyStatusPages = "legacyPages"; + clusterView.services = Collections.singletonList(serviceView); + applicationView.clusters = Collections.singletonList(clusterView); + response.setResponse(applicationView); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + response.render(stream); + Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray()); + Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response.json")).getBytes(StandardCharsets.UTF_8)); + + assertEquals("service-api-response.json", + new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), + new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8)); + } + + @Test + public void testServiceViewResponseWithURLs() throws URISyntaxException, IOException { + ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.prod, RegionName.from("us-west-1")), + ApplicationId.from("tenant2", "application2", "default"), + Collections.singletonList(new URI("http://cfg1.test/")), + new URI("http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1")); + ApplicationView applicationView = new ApplicationView(); + ClusterView clusterView = new ClusterView(); + clusterView.type = "container"; + clusterView.name = "cluster1"; + clusterView.url = "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health"; + ServiceView serviceView = new ServiceView(); + serviceView.url = null; + serviceView.serviceType = "container"; + serviceView.serviceName = "service1"; + serviceView.configId = "configId1"; + serviceView.host = "host1"; + serviceView.legacyStatusPages = "legacyPages"; + clusterView.services = Collections.singletonList(serviceView); + applicationView.clusters = Collections.singletonList(clusterView); + response.setResponse(applicationView); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + response.render(stream); + Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray()); + Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response-with-urls.json")).getBytes(StandardCharsets.UTF_8)); + + assertEquals("service-api-response.json", + new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8), + new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json new file mode 100644 index 00000000000..f74b2290d81 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json @@ -0,0 +1 @@ +{"message":"Triggered deployment of application 'tenant1.application1' on version 6.1"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json new file mode 100644 index 00000000000..ecee1c8dbde --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json @@ -0,0 +1,3 @@ +[ + @include(application-reference.json) +]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json new file mode 100644 index 00000000000..1ec229a2b4a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json @@ -0,0 +1,5 @@ +{ + "application":"application1", + "instance":"default", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json new file mode 100644 index 00000000000..5df690f5bc7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -0,0 +1,104 @@ +{ + "deploying": { + "version": "(ignore)" + }, + "deploymentJobs": [ + { + "type": "system-test", + "success": true, + "lastTriggered": { + "version": "(ignore)", + "at": "(ignore)" + }, + "lastCompleted": { + "version": "(ignore)", + "at": "(ignore)" + }, + "lastSuccess": { + "version": "(ignore)", + "at": "(ignore)" + } + }, + { + "type":"staging-test", + "success":true, + "lastTriggered":{ + "version":"(ignore)", + "at":"(ignore)" + }, + "lastCompleted":{ + "version":"(ignore)", + "at":"(ignore)" + }, + "lastSuccess":{ + "version":"(ignore)", + "at":"(ignore)" + } + }, + { + "type":"production-corp-us-east-1", + "success":false, + "lastTriggered":{ + "version":"(ignore)", + "revision":{ + "hash":"(ignore)", + "source":{ + "gitRepository":"repository1", + "gitBranch":"master", + "gitCommit":"commit1" + } + }, + "at":"(ignore)" + }, + "lastCompleted":{ + "version":"(ignore)", + "revision":{ + "hash":"(ignore)", + "source":{ + "gitRepository":"repository1", + "gitBranch":"master", + "gitCommit":"commit1" + } + }, + "at":"(ignore)" + }, + "firstFailing":{ + "version":"(ignore)", + "revision":{ + "hash":"(ignore)", + "source":{ + "gitRepository":"repository1", + "gitBranch":"master", + "gitCommit":"commit1" + } + }, + "at":"(ignore)" + } + } + ], + "compileVersion": "(ignore)", + "globalRotations": [ + "http://fake-global-rotation-tenant1.application1" + ], + "instances": [ + { + "environment": "dev", + "region": "us-west-1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default" + }, + { + "environment": "prod", + "region": "corp-us-east-1", + "instance": "default", + "bcpStatus": {"rotationStatus":"UNKNOWN"}, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default" + } + ] + , + "metrics": + { + "queryServiceQuality":0.5, + "writeServiceQuality":0.7 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json new file mode 100644 index 00000000000..3a1cc9c6582 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json @@ -0,0 +1,5 @@ +{ + "data": [ + "domain1" + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json new file mode 100644 index 00000000000..acfb67b702b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json @@ -0,0 +1,3 @@ +{ + "generation": 1 +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json new file mode 100644 index 00000000000..3c428332aa6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json @@ -0,0 +1,3 @@ +{ + "shouldRefreshCookie":true +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json new file mode 100644 index 00000000000..709548e87a7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json @@ -0,0 +1,3 @@ +{ + "message":"Created user 'newuser'" +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json new file mode 100644 index 00000000000..f9496edecde --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json @@ -0,0 +1,4 @@ +{ + "error-code":"BAD_REQUEST", + "message":"Failed to prepare application" +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json new file mode 100644 index 00000000000..0de6465156f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json @@ -0,0 +1,4 @@ +{ + "error-code":"INVALID_APPLICATION_PACKAGE", + "message":"Failed to prepare application" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json new file mode 100644 index 00000000000..669df626378 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json @@ -0,0 +1,4 @@ +{ + "error-code":"OUT_OF_CAPACITY", + "message":"Failed to prepare application" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json new file mode 100644 index 00000000000..d1ae5253a00 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json @@ -0,0 +1,9 @@ +{ + "revisionId":"(ignore)", + "applicationZipSize":412, + "prepareMessages":[], + "configChangeActions":{ + "restart":[], + "refeed":[] + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json new file mode 100644 index 00000000000..50599581f92 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -0,0 +1,57 @@ +{ + "serviceUrls": [ + "qrs-endpoint","feeding-endpoint","global-endpoint","alias-endpoint" + ], + "nodes": "http://localhost:8080/zone/v2/prod/corp-us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "elkUrl": "http://log.prod.corp-us-east-1.test/#/discover?_g=()&_a=(columns:!(_source),index:'logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'HV-tenant:%22tenant1%22%20AND%20HV-application:%22application1%22%20AND%20HV-region:%22corp-us-east-1%22%20AND%20HV-instance:%22default%22%20AND%20HV-environment:%22prod%22')),sort:!('@timestamp',desc))", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=corp-us-east-1&application=tenant1.application1", + "version": "(ignore)", + "revision": "(ignore)", + "deployTimeEpochMs": "(ignore)", + "screwdriverId":"123", + "gitRepository":"repository1", + "gitBranch":"master", + "gitCommit":"commit1", + "cost": { + "zone": "prod.us-west-1", + "tenant": "tenant1", + "app": "application1.default", + "tco": 37, + "utilization": 1.0, + "waste": 0.0, + "cluster": { + "cluster1": { + "count": 2, + "resource": "cpu", + "utilization": 1.0, + "tco": 25, + "flavor": "flavor1", + "waste": 10, + "type": "content", + "util": { + "cpu": 0.0, + "mem": 0.0, + "disk": 0.0, + "diskBusy": 0.0 + }, + "usage": { + "cpu": 0.0, + "mem": 0.0, + "disk": 0.0, + "diskBusy": 0.0 + }, + "hostnames": [ + "host1", + "host2" + ] + } + } + }, + "metrics": { + "queriesPerSecond":1.0, + "writesPerSecond":2.0, + "documentCount":3.0, + "queryLatencyMillis":4.0, + "writeLatencyMillis":5.0 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json new file mode 100644 index 00000000000..1ed86920fdf --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json @@ -0,0 +1 @@ +{"message":"Rotations [global-endpoint, alias-endpoint] successfully set to in service"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json new file mode 100644 index 00000000000..31a337f7706 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json @@ -0,0 +1 @@ +{"globalrotationoverride":["global-endpoint",{"status":"in","reason":"","agent":"","timestamp":1497618757},"alias-endpoint",{"status":"in","reason":"","agent":"","timestamp":1497618757}]}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json new file mode 100644 index 00000000000..eefd2859241 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json @@ -0,0 +1 @@ +{"message":"Rotations [global-endpoint, alias-endpoint] successfully set to out of service"}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json new file mode 100644 index 00000000000..530e21c6c7a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json @@ -0,0 +1,5 @@ +{ + "bcpStatus": { + "rotationStatus": "IN" + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json new file mode 100644 index 00000000000..9e26dfeeb6e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json @@ -0,0 +1 @@ +{}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json new file mode 100644 index 00000000000..8de85754ab0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json @@ -0,0 +1,9 @@ +{ + "type": "OPSDB", + "property": "property1", + "propertyId": "1234", + "userGroup": "group1", + "applications": [ + + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json new file mode 100644 index 00000000000..9f0a7ec603e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json @@ -0,0 +1,9 @@ +{ + "type": "OPSDB", + "property": "property2", + "propertyId": "4321", + "userGroup": "group1", + "applications": [ + + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json new file mode 100644 index 00000000000..596dea037bd --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json @@ -0,0 +1,6 @@ +{ + "properties": [ + {"propertyid": "1234", "property": "foo"}, + {"propertyid": "4321", "property": "bar"} + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json new file mode 100644 index 00000000000..6e4e319d3e1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json @@ -0,0 +1,22 @@ +{ + "resources":[ + { + "url":"http://localhost:8080/application/v4/user/" + }, + { + "url":"http://localhost:8080/application/v4/tenant/" + }, + { + "url":"http://localhost:8080/application/v4/tenant-pipeline/" + }, + { + "url":"http://localhost:8080/application/v4/athensDomain/" + }, + { + "url":"http://localhost:8080/application/v4/property/" + }, + { + "url":"http://localhost:8080/application/v4/cookiefreshness/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json new file mode 100644 index 00000000000..0e610c4d4b2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json @@ -0,0 +1,18 @@ +{ + "clusters": [ + { + "name": "cluster1", + "type": "container", + "url": "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health", + "services": [ + { + "url": null, + "serviceType": "container", + "serviceName": "service1", + "configId": "configId1", + "host": "host1" + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json new file mode 100644 index 00000000000..3380eb26911 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json @@ -0,0 +1,18 @@ +{ + "clusters": [ + { + "name": "cluster1", + "type": "container", + "url": "cluster-url", + "services": [ + { + "url": null, + "serviceType": "container", + "serviceName": "service1", + "configId": "configId1", + "host": "host1" + } + ] + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json new file mode 100644 index 00000000000..8fb64d65ff8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config" + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json new file mode 100644 index 00000000000..8a0849393c9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json @@ -0,0 +1,18 @@ +{ + "clusters": [ + { + "name": "cluster1", + "type": "content", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", + "services": [ + { + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", + "serviceType": "storagenode", + "serviceName": "storagenode", + "configId": "cluster1/storage/0", + "host": "host1" + } + ] + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json new file mode 100644 index 00000000000..a9d9cd33ae8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json @@ -0,0 +1,11 @@ +[ + { + "tenant": "tenant1", + "metaData": { + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1" + }, + "url": "http://localhost:8080/application/v4/tenant/tenant1" + } +]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json new file mode 100644 index 00000000000..4e6fef1b994 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json @@ -0,0 +1,13 @@ +{ + "tenantPipelines": [ + { + "screwdriverId": "123", + "tenant": "tenant1", + "application": "application1", + "instance": "default" + } + ], + "brokenTenantPipelines": [ + + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json new file mode 100644 index 00000000000..87901218c2e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -0,0 +1,12 @@ +{ + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1", + "applications": [ + { + "application":"application1", + "instance":"default", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json new file mode 100644 index 00000000000..3deef01bb44 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json @@ -0,0 +1,9 @@ +{ + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property2", + "propertyId": "1234", + "applications": [ + + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json new file mode 100644 index 00000000000..88ec5ec7d3d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json @@ -0,0 +1,8 @@ +{ + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1", + "applications": [ + + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json new file mode 100644 index 00000000000..d3927cbcfcf --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json @@ -0,0 +1,5 @@ +{ + "user": "mytenant", + "tenants": @include(tenant-list.json), + "tenantExists": false +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java new file mode 100644 index 00000000000..011bbadb91c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.controller; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * @author bratseth + */ +public class ControllerApiTest extends ControllerContainerTest { + + private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/"; + + @Test + public void testControllerApi() throws IOException { + ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + + tester.assertResponse(new Request("http://localhost:8080/controller/v1/"), new File("root.json")); + + // POST deactivation of a maintenance job + assertResponse(new Request("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", + new byte[0], Request.Method.POST), + 200, + "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}"); + // GET a list of all maintenance jobs + tester.assertResponse(new Request("http://localhost:8080/controller/v1/maintenance/"), + new File("maintenance.json")); + // DELETE deactivation of a maintenance job + assertResponse(new Request("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", + new byte[0], Request.Method.DELETE), + 200, + "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}"); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json new file mode 100644 index 00000000000..d8ca5e59b4f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -0,0 +1,31 @@ +{ + "jobs":[ + { + "name":"DelayedDeployer" + }, + { + "name":"Upgrader" + }, + { + "name":"FailureRedeployer" + }, + { + "name":"DeploymentExpirer" + }, + { + "name":"MetricsReporter" + }, + { + "name":"VersionStatusUpdater" + }, + { + "name":"DeploymentIssueReporter" + }, + { + "name":"OutstandingChangeDeployer" + } + ], + "inactive":[ + "DeploymentExpirer" + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json new file mode 100644 index 00000000000..155c13fd5ed --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json @@ -0,0 +1,7 @@ +{ + "resources":[ + { + "url":"http://localhost:8080/controller/v1/maintenance/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java new file mode 100644 index 00000000000..26741148d3a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -0,0 +1,72 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.deployment; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; + +/** + * @author bratseth + */ +public class DeploymentApiTest extends ControllerContainerTest { + + private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/"; + + @Test + public void testDeploymentApi() throws IOException { + ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + tester.containerTester().updateSystemVersion(); + long projectId = 11; + Application app = tester.createApplication(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + tester.notifyJobCompletion(app.id(), projectId, true, component); + tester.deploy(app, applicationPackage, new Zone(Environment.test, RegionName.from("us-east-1")), projectId); + tester.notifyJobCompletion(app.id(), projectId, true, systemTest); + tester.deploy(app, applicationPackage, new Zone(Environment.staging, RegionName.from("us-east-3")), projectId); + tester.notifyJobCompletion(app.id(), projectId, false, stagingTest); + + tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()), + tester.controller())); + tester.assertResponse(new Request("http://localhost:8080/deployment/v1/"), + new File("root.json")); + } + + private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) { + List<VespaVersion> censored = new ArrayList<>(); + for (VespaVersion version : versionStatus.versions()) { + if ( ! version.configServerHostnames().isEmpty()) + version = new VespaVersion(version.statistics(), + version.releaseCommit(), + version.releasedAt(), + version.isCurrentSystemVersion(), + ImmutableSet.of("config1.test", "config2.test"), + controller); + censored.add(version); + } + return new VersionStatus(censored); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json new file mode 100644 index 00000000000..4ea1359519f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -0,0 +1,50 @@ +{ + "versions":[ + { + "version":"(ignore)", + "confidence":"normal", + "commit":"(ignore)", + "date":0, + "controllerVersion":false, + "systemVersion":true, + "configServers":[ + { + "hostname":"config1.test" + }, + { + "hostname":"config2.test" + } + ], + "failingApplications":[ + { + "tenant":"tenant1", + "application":"application1", + "instance":"default", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "failingSince": "(ignore)" + } + ], + "productionApplications":[ + + ] + }, + { + "version":"(ignore)", + "confidence":"normal", + "commit":"(ignore)", + "date":0, + "controllerVersion":true, + "systemVersion":false, + "configServers":[ + + ], + "failingApplications":[ + + ], + "productionApplications":[ + + ] + } + ] +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java new file mode 100644 index 00000000000..0c31c6e2cc5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java @@ -0,0 +1,79 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig; +import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig.Builder; +import org.junit.Test; + +import java.util.Arrays; + +import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS; +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS; +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author gjoranv + */ +public class AccessControlRequestFilterTest { + + @Test + public void any_options_request_yields_access_control_headers_in_response() { + HeaderFields headers = doFilterRequest(newRequestFilter(), "http://any.origin"); + ACCESS_CONTROL_HEADERS.keySet().forEach( + header -> assertFalse("Empty header: " + header, headers.getFirst(header).isEmpty())); + } + + @Test + public void allowed_request_origin_yields_allow_origin_header_in_response() { + final String ALLOWED_ORIGIN = "http://allowed.origin"; + HeaderFields headers = doFilterRequest(newRequestFilter(ALLOWED_ORIGIN), ALLOWED_ORIGIN); + assertEquals(ALLOWED_ORIGIN, headers.getFirst(ALLOW_ORIGIN_HEADER)); + } + + @Test + public void disallowed_request_origin_does_not_yield_allow_origin_header_in_response() { + HeaderFields headers = doFilterRequest(newRequestFilter("http://allowed.origin"), "http://disallowed.origin"); + assertNull(headers.getFirst(ALLOW_ORIGIN_HEADER)); + } + + private static HeaderFields doFilterRequest(SecurityRequestFilter filter, String originUrl) { + AccessControlResponseHandler responseHandler = new AccessControlResponseHandler(); + filter.filter(newOptionsRequest(originUrl), responseHandler); + return responseHandler.response.headers(); + } + + private static DiscFilterRequest newOptionsRequest(String origin) { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getHeader("Origin")).thenReturn(origin); + when(request.getMethod()).thenReturn(OPTIONS.name()); + return request; + } + + private static AccessControlRequestFilter newRequestFilter(String... allowedOriginUrls) { + Builder builder = new Builder(); + Arrays.asList(allowedOriginUrls).forEach(builder::allowedUrls); + return new AccessControlRequestFilter(new HttpAccessControlConfig(builder)); + } + + private static class AccessControlResponseHandler implements ResponseHandler { + Response response; + + @Override + public ContentChannel handleResponse(Response response) { + this.response = response; + return mock(ContentChannel.class); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java new file mode 100644 index 00000000000..1b368d0a4b8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java @@ -0,0 +1,112 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.jdisc.http.Cookie; +import com.yahoo.jdisc.http.filter.DiscFilterResponse; +import com.yahoo.jdisc.http.filter.RequestView; +import com.yahoo.jdisc.http.filter.SecurityResponseFilter; +import com.yahoo.jdisc.http.servlet.ServletOrJdiscHttpResponse; +import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig; +import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig.Builder; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS; +import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author gjoranv + */ +public class AccessControlResponseFilterTest { + + @Test + public void any_request_yields_access_control_headers_in_response() { + Map<String, String> headers = doFilterRequest(newResponseFilter(), "http://any.origin"); + ACCESS_CONTROL_HEADERS.keySet().forEach( + header -> assertFalse("Empty header: " + header, headers.get(header).isEmpty())); + } + + @Test + public void allowed_request_origin_yields_allow_origin_header_in_response() { + final String ALLOWED_ORIGIN = "http://allowed.origin"; + Map<String, String> headers = doFilterRequest(newResponseFilter(ALLOWED_ORIGIN), ALLOWED_ORIGIN); + assertEquals(ALLOWED_ORIGIN, headers.get(ALLOW_ORIGIN_HEADER)); + } + + @Test + public void disallowed_request_origin_does_not_yield_allow_origin_header_in_response() { + Map<String, String> headers = doFilterRequest(newResponseFilter("http://allowed.origin"), "http://disallowed.origin"); + assertNull(headers.get(ALLOW_ORIGIN_HEADER)); + } + + @Test + public void any_request_origin_yields_allow_origin_header_in_response_when_wildcard_is_allowed() { + Map<String, String> headers = doFilterRequest(newResponseFilter("*"), "http://any.origin"); + assertEquals("*", headers.get(ALLOW_ORIGIN_HEADER)); + } + + private static Map<String, String> doFilterRequest(SecurityResponseFilter filter, String originUrl) { + TestResponse response = new TestResponse(); + filter.filter(response, newRequestView(originUrl)); + return Collections.unmodifiableMap(response.headers); + } + + private static AccessControlResponseFilter newResponseFilter(String... allowedOriginUrls) { + Builder builder = new Builder(); + Arrays.asList(allowedOriginUrls).forEach(builder::allowedUrls); + return new AccessControlResponseFilter(new HttpAccessControlConfig(builder)); + } + + private static RequestView newRequestView(String originUrl) { + RequestView request = mock(RequestView.class); + when(request.getFirstHeader("Origin")).thenReturn(Optional.of(originUrl)); + return request; + } + + private static class TestResponse extends DiscFilterResponse { + Map<String, String> headers = new HashMap<>(); + + TestResponse() { + super(mock(ServletOrJdiscHttpResponse.class)); + } + + @Override + public void setHeader(String name, String value) { + headers.put(name, value); + } + + @Override + public String getHeader(String name) { + return headers.get(name); + } + + @Override + public void removeHeaders(String s) { throw new UnsupportedOperationException(); } + + @Override + public void setHeaders(String s, String s1) { throw new UnsupportedOperationException(); } + + @Override + public void setHeaders(String s, List<String> list) { throw new UnsupportedOperationException(); } + + @Override + public void addHeader(String s, String s1) { throw new UnsupportedOperationException(); } + + @Override + public void setCookies(List<Cookie> list) { throw new UnsupportedOperationException(); } + + @Override + public void setStatus(int i) { throw new UnsupportedOperationException(); } + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json new file mode 100644 index 00000000000..90b1b027529 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json @@ -0,0 +1,41 @@ +{ + "services":[ + { + "name":"provision", + "url":"http://localhost:8080/provision/v1/", + "wadl":"http://localhost:8080/provision/application.wadl" + }, + { + "name":"statuspage", + "url":"http://localhost:8080/statuspage/v1/", + "wadl":"http://localhost:8080/statuspage/application.wadl" + }, + { + "name":"zone", + "url":"http://localhost:8080/zone/v1/", + "wadl":"http://localhost:8080/zone/application.wadl" + }, + { + "name":"zone", + "url":"http://localhost:8080/zone/v2/", + "wadl":"http://localhost:8080/zone/application.wadl" + }, + { + "name":"cost", + "url":"http://localhost:8080/cost/v1/", + "wadl":"http://localhost:8080/cost/application.wadl" + }, + { + "name":"application", + "url":"http://localhost:8080/application/v4/" + }, + { + "name":"deployment", + "url":"http://localhost:8080/deployment/v1/" + }, + { + "name":"screwdriver", + "url":"http://localhost:8080/screwdriver/v1/release/vespa" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java new file mode 100644 index 00000000000..bdfd0f9794f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -0,0 +1,165 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.screwdriver; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; +import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static junit.framework.TestCase.assertNotNull; +import static junit.framework.TestCase.assertNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + * @author jvenstad + */ +public class ScrewdriverApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/"; + private static final Zone testZone = new Zone(Environment.test, RegionName.from("us-east-1")); + private static final Zone stagingZone = new Zone(Environment.staging, RegionName.from("us-east-3")); + + @Test + public void testGetReleaseStatus() throws Exception { + ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/release/vespa"), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Information about the current system version is not available at this time\"}", + 404); + + tester.controller().updateVersionStatus(VersionStatus.compute(tester.controller())); + tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/release/vespa"), + new File("release-response.json"), 200); + } + + @Test + public void testJobStatusReporting() throws Exception { + ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + tester.containerTester().updateSystemVersion(); + long projectId = 1; + Application app = tester.createApplication(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + + Version vespaVersion = new Version("6.1"); // system version from mock config server client + + // Make web service calls. + notifyCompletion(app.id(), projectId, JobType.component, Optional.empty()); + tester.deploy(app, applicationPackage, testZone, projectId); + notifyCompletion(app.id(), projectId, JobType.systemTest, Optional.empty()); + + // Notifying about unknown job fails + tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport", + jsonReport(app.id(), JobType.productionUsEast3, projectId, 1L, + Optional.empty(), false, true) + .getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + new File("unexpected-completion.json"), 400); + + // ... and assert it was recorded + JobStatus recordedStatus = + tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(JobType.component); + + assertNotNull("Status was recorded", recordedStatus); + assertTrue(recordedStatus.isSuccess()); + assertEquals(vespaVersion, recordedStatus.lastCompleted().get().version()); + + recordedStatus = + tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(JobType.productionApNortheast2); + assertNull("Status of never-triggered jobs is empty", recordedStatus); + + Response response; + + response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.GET)); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); + + // Check that GET didn't affect the enqueued jobs. + response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE)); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); + + Thread.sleep(50); + // Check that the *first* DELETE has removed the enqueued jobs. + assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE), + 200, "[]"); + } + + @Test + public void testJobStatusReportingOutOfCapacity() throws Exception { + ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + tester.containerTester().updateSystemVersion(); + + long projectId = 1; + Application app = tester.createApplication(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + + // Report job failing with out of capacity + notifyCompletion(app.id(), projectId, JobType.component, Optional.empty()); + tester.deploy(app, applicationPackage, testZone, projectId); + notifyCompletion(app.id(), projectId, JobType.systemTest, Optional.empty()); + tester.deploy(app, applicationPackage, stagingZone, projectId); + notifyCompletion(app.id(), projectId, JobType.stagingTest, Optional.of(JobError.outOfCapacity)); + + // Appropriate error is recorded + JobStatus jobStatus = tester.controller().applications().get(app.id()) + .get() + .deploymentJobs() + .jobStatus() + .get(JobType.stagingTest); + assertFalse(jobStatus.isSuccess()); + assertEquals(JobError.outOfCapacity, jobStatus.jobError().get()); + } + + private void notifyCompletion(ApplicationId app, long projectId, JobType jobType, Optional<JobError> error) throws IOException { + assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport", + jsonReport(app, jobType, projectId, 1L, error, false, true).getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + 200, "ok"); + } + + private static String jsonReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber, + Optional<JobError> jobError, boolean selfTriggering, boolean gitChanges) { + return + "{\n" + + " \"projectId\" : " + projectId + ",\n" + + " \"jobName\" :\"" + jobType.id() + "\",\n" + + " \"buildNumber\" : " + buildNumber + ",\n" + + jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + + " \"selfTriggering\": " + selfTriggering + ",\n" + + " \"gitChanges\" : " + gitChanges + ",\n" + + " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + + " \"application\" :\"" + applicationId.application().value() + "\",\n" + + " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + + "}"; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json new file mode 100644 index 00000000000..9d96e08e695 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json @@ -0,0 +1,5 @@ +{ + "version":"(ignore)", + "sha":"(ignore)", + "date":0 +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json new file mode 100644 index 00000000000..e293d85b594 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json @@ -0,0 +1,4 @@ +{ + "error-code": "BAD_REQUEST", + "message": "Got notified about completion of job status of productionUsEast3[ last triggered: (never), last completed: (never), first failing: (not failing), lastSuccess: (never)], but that has not been triggered nor deployed" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java new file mode 100644 index 00000000000..5a89ebb2d3f --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author bratseth + */ +public class MockRoutingGenerator implements RoutingGenerator { + + @Override + public List<RoutingEndpoint> endpoints(DeploymentId deployment) { + List<RoutingEndpoint> endpoints = new ArrayList<>(); + endpoints.add(new RoutingEndpoint("qrs-endpoint", false)); + endpoints.add(new RoutingEndpoint("feeding-endpoint", false)); + endpoints.add(new RoutingEndpoint("global-endpoint", true)); + endpoints.add(new RoutingEndpoint("alias-endpoint", true)); + return endpoints; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java new file mode 100644 index 00000000000..11e55edb5a5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -0,0 +1,283 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Test; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test computing of version status + * + * @author bratseth + */ +public class VersionStatusTest { + + @Test + public void testEmptyVersionStatus() { + VersionStatus status = VersionStatus.empty(); + assertFalse(status.systemVersion().isPresent()); + assertTrue(status.versions().isEmpty()); + } + + @Test + public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() { + ControllerTester tester = new ControllerTester(); + Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1); + tester.configServerClientMock().setDefaultConfigServerVersion(largerThanCurrent); + VersionStatus versionStatus = VersionStatus.compute(tester.controller()); + assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber()); + } + + @Test + public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException { + ControllerTester tester = new ControllerTester(); + Version oldest = new Version(5); + tester.configServerClientMock().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); + VersionStatus versionStatus = VersionStatus.compute(tester.controller()); + assertEquals(oldest, versionStatus.systemVersion().get().versionNumber()); + } + + @Test + public void testVersionStatusAfterApplicationUpdates() { + DeploymentTester tester = new DeploymentTester(); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .region("us-east-3") + .build(); + + // Application versions which are older than the current version + Version version1 = new Version("5.1"); + Version version2 = new Version("5.2"); + tester.upgradeSystem(version1); + + // Setup applications + Application app1 = tester.createAndDeploy("app1", 11, applicationPackage); + Application app2 = tester.createAndDeploy("app2", 22, applicationPackage); + Application app3 = tester.createAndDeploy("app3", 33, applicationPackage); + + // version2 is released + tester.upgradeSystem(version2); + + // - app1 is in production on version1, but then fails in system test on version2 + tester.completeUpgradeWithError(app1, version2, applicationPackage, systemTest); + // - app2 is partially in production on version1 and partially on version2 + tester.completeUpgradeWithError(app2, version2, applicationPackage, productionUsEast3); + // - app3 is in production on version1, but then fails in staging test on version2 + tester.completeUpgradeWithError(app3, version2, applicationPackage, stagingTest); + + VersionStatus versionStatus = VersionStatus.compute(tester.controller()); + List<VespaVersion> versions = versionStatus.versions(); + assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size()); + + VespaVersion v0 = versions.get(2); + assertEquals(tester.configServerClientMock().getDefaultConfigServerVersion(), v0.versionNumber()); + assertEquals(0, v0.statistics().failing().size()); + assertEquals(0, v0.statistics().production().size()); + + VespaVersion v1 = versions.get(0); + assertEquals(version1, v1.versionNumber()); + assertEquals(0, v1.statistics().failing().size()); + // All applications are on v1 in at least one zone + assertEquals(3, v1.statistics().production().size()); + assertTrue(v1.statistics().production().contains(app2.id())); + assertTrue(v1.statistics().production().contains(app1.id())); + + VespaVersion v2 = versions.get(1); + assertEquals(version2, v2.versionNumber()); + // All applications have failed on v2 in at least one zone + assertEquals(3, v2.statistics().failing().size()); + assertTrue(v2.statistics().failing().contains(app1.id())); + assertTrue(v2.statistics().failing().contains(app3.id())); + // Only one application is on v2 in at least one zone + assertEquals(1, v2.statistics().production().size()); + assertTrue(v2.statistics().production().contains(app2.id())); + + VespaVersion v3 = versions.get(3); + assertEquals(Vtag.currentVersion, v3.versionNumber()); + assertEquals(0, v3.statistics().failing().size()); + assertEquals(0, v3.statistics().production().size()); + } + + @Test + public void testVersionConfidence() { + DeploymentTester tester = new DeploymentTester(); + + Version version0 = new Version("5.0"); + tester.upgradeSystem(version0); + + // Setup applications + Application canary0 = tester.createAndDeploy("canary0", 0, "canary"); + Application canary1 = tester.createAndDeploy("canary1", 1, "canary"); + Application canary2 = tester.createAndDeploy("canary2", 2, "canary"); + Application default0 = tester.createAndDeploy("default0", 00, "default"); + Application default1 = tester.createAndDeploy("default1", 11, "default"); + Application default2 = tester.createAndDeploy("default2", 22, "default"); + Application default3 = tester.createAndDeploy("default3", 33, "default"); + Application default4 = tester.createAndDeploy("default4", 44, "default"); + Application default5 = tester.createAndDeploy("default5", 55, "default"); + Application default6 = tester.createAndDeploy("default6", 66, "default"); + Application default7 = tester.createAndDeploy("default7", 77, "default"); + Application default8 = tester.createAndDeploy("default8", 88, "default"); + Application default9 = tester.createAndDeploy("default9", 99, "default"); + Application conservative0 = tester.createAndDeploy("conservative1", 000, "conservative"); + + + // The following applications should not affect confidence calculation: + + // Application without deployment + Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L); + + // Pull request build + Application ignored1 = tester.controllerTester().createApplication(new TenantId("tenant1"), + "ignored1", + "default-pr42", 1000); + + Version version1 = new Version("5.1"); + Version version2 = new Version("5.2"); + tester.upgradeSystem(version1); + + // Canaries upgrade to new versions and fail + tester.completeUpgrade(canary0, version1, "canary"); + tester.completeUpgradeWithError(canary1, version1, "canary", productionUsWest1); + tester.upgradeSystem(version2); + tester.completeUpgrade(canary2, version2, "canary"); + + VersionStatus versionStatus = VersionStatus.compute(tester.controller()); + List<VespaVersion> versions = versionStatus.versions(); + + assertEquals("One canary failed: Broken", + VespaVersion.Confidence.broken, confidence(versions, version1)); + assertEquals("Nothing has failed but not all canaries has deployed: Low", + VespaVersion.Confidence.low, confidence(versions, version2)); + assertEquals("Current version of this - no deployments: Low", + VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + + // All canaries are upgraded to version2 which raises confidence to normal and more apps upgrade + tester.completeUpgrade(canary0, version2, "canary"); + tester.completeUpgrade(canary1, version2, "canary"); + tester.upgradeSystem(version2); + tester.completeUpgrade(default0, version2, "default"); + tester.completeUpgrade(default1, version2, "default"); + tester.completeUpgrade(default2, version2, "default"); + tester.completeUpgrade(default3, version2, "default"); + tester.completeUpgrade(default4, version2, "default"); + tester.completeUpgrade(default5, version2, "default"); + tester.completeUpgrade(default6, version2, "default"); + tester.completeUpgrade(default7, version2, "default"); + + versionStatus = VersionStatus.compute(tester.controller()); + versions = versionStatus.versions(); + + assertEquals("No deployments: Low", + VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("All canaries deployed + < 90% of defaults: Normal", + VespaVersion.Confidence.normal, confidence(versions, version2)); + assertEquals("Current version of this - no deployments: Low", + VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + + // Another default application upgrades, raising confidence to high + tester.completeUpgrade(default8, version2, "default"); + + versionStatus = VersionStatus.compute(tester.controller()); + versions = versionStatus.versions(); + + assertEquals("No deployments: Low", + VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("90% of defaults deployed successfully: High", + VespaVersion.Confidence.high, confidence(versions, version2)); + assertEquals("Current version of this - no deployments: Low", + VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + + // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version + // as broken + Version version3 = new Version("5.3"); + tester.upgradeSystem(version3); + tester.completeUpgrade(canary0, version3, "canary"); + tester.completeUpgrade(canary1, version3, "canary"); + tester.completeUpgrade(canary2, version3, "canary"); + tester.upgradeSystem(version3); + tester.completeUpgradeWithError(default0, version3, "default", stagingTest); + tester.completeUpgradeWithError(default1, version3, "default", stagingTest); + tester.completeUpgradeWithError(default2, version3, "default", stagingTest); + tester.completeUpgradeWithError(default9, version3, "default", stagingTest); + + versionStatus = VersionStatus.compute(tester.controller()); + versions = versionStatus.versions(); + + assertEquals("No deployments: Low", + VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("40% of defaults failed: Broken", + VespaVersion.Confidence.broken, confidence(versions, version3)); + assertEquals("Current version of this - no deployments: Low", + VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + } + + @Test + public void testComputeIgnoresVersionWithUnknownGitMetadata() { + ControllerTester tester = new ControllerTester(); + ApplicationController applications = tester.controller().applications(); + + tester.gitHubClientMock() + .mockAny(false) + .knownTag(Vtag.currentVersion.toFullString(), "foo") // controller + .knownTag("6.1.0", "bar"); // config server + + Version versionWithUnknownTag = new Version("6.1.2"); + + Application app = tester.createAndDeploy("tenant1", "domain1","application1", Environment.test, 11); + applications.notifyJobCompletion(mockReport(app, component, true)); + applications.notifyJobCompletion(mockReport(app, systemTest, true)); + + List<VespaVersion> vespaVersions = VersionStatus.compute(tester.controller()).versions(); + + assertEquals(2, vespaVersions.size()); // controller and config server + assertTrue("Version referencing unknown tag is skipped", + vespaVersions.stream().noneMatch(v -> v.versionNumber().equals(versionWithUnknownTag))); + } + + private VespaVersion.Confidence confidence(List<VespaVersion> versions, Version version) { + return versions.stream() + .filter(v -> v.statistics().version().equals(version)) + .findFirst() + .map(VespaVersion::confidence) + .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version)); + } + + private DeploymentJobs.JobReport mockReport(Application application, DeploymentJobs.JobType jobType, boolean success) { + return new DeploymentJobs.JobReport( + application.id(), + jobType, + application.deploymentJobs().projectId().get(), + 1L, + JobError.from(success), + false, + true + ); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java new file mode 100644 index 00000000000..4e2e4bb15b4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.restapi.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.UriBuilder; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author frodelu + */ +public class StatusPageResourceTest { + + private StatusPageResource statusPage; + + @Before + public void setup() throws IOException { + + Client mockClient = Mockito.mock(Client.class); + WebTarget mockTarget = Mockito.mock(WebTarget.class); + Invocation.Builder mockRequest = Mockito.mock(Invocation.Builder.class); + KeyService keyService = Mockito.mock(KeyService.class); + + Mockito.when(mockClient.target(Mockito.any(UriBuilder.class))).thenReturn(mockTarget); + Mockito.when(mockTarget.request()).thenReturn(mockRequest); + Mockito.when(mockRequest.get(JsonNode.class)).thenReturn( + new ObjectMapper().readTree("{\"page\":{\"name\":\"Vespa\"}}")); + Mockito.when(keyService.getSecret(Mockito.any(String.class))).thenReturn("testpage:testkey"); + + statusPage = new StatusPageResource(keyService, mockClient); + } + + + @Test + public void default_url() { + UriBuilder uri = statusPage.statusPageURL("incidents", null); + assertNotNull("URI not initialized", uri); + assertEquals("https://testpage.statuspage.io/api/v2/incidents.json?api_key=testkey", uri.toTemplate()); + } + + @Test + public void url_with_since_param() { + UriBuilder uri = statusPage.statusPageURL("incidents", "2015-01-01T00:00+00:00"); + assertNotNull("URI not initialized", uri); + assertEquals("https://testpage.statuspage.io/api/v2/incidents.json?api_key=testkey&since=2015-01-01T00%3A00%2B00%3A00", uri.toTemplate()); + } + + @Test + public void valid_status_page() { + JsonNode result = statusPage.statusPage("incidents", null); + assertNotNull("No result from StatusPage.io", result); + assertEquals("Vespa", result.get("page").get("name").asText()); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java new file mode 100644 index 00000000000..8fc218a9e8b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java @@ -0,0 +1,200 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.rotation; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; +import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; +import com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.StringReader; +import java.net.URI; +import java.util.Collections; +import java.util.Set; + +import static org.fest.assertions.Assertions.*; + +/** + * @author Oyvind Gronnesby + */ +public class ControllerRotationRepositoryTest { + + private final RotationsConfig rotationsConfig = new RotationsConfig( + new RotationsConfig.Builder() + .rotations("foo-1", "foo-1.com") + .rotations("foo-2", "foo-2.com") + ); + private final RotationsConfig rotationsConfigWhitespaces = new RotationsConfig( + new RotationsConfig.Builder() + .rotations("foo-1", "\n foo-1.com \n") + .rotations("foo-2", "foo-2.com") + ); + private final ControllerDb controllerDb = new MemoryControllerDb(); + private final ApplicationId applicationId = ApplicationId.from("msbe", "tumblr-search", "default"); + + @Rule public ExpectedException thrown = ExpectedException.none(); + + private final DeploymentSpec deploymentSpec = DeploymentSpec.fromXml( + new StringReader( + "<deployment>" + + " <prod global-service-id='foo'>" + + " <region active='true'>us-east</region>" + + " <region active='true'>us-west</region>" + + " </prod>" + + "</deployment>" + ) + ); + + private final DeploymentSpec deploymentSpecOneRegion = DeploymentSpec.fromXml( + new StringReader( + "<deployment>" + + " <prod global-service-id='nalle'>" + + " <region active='true'>us-east</region>" + + " </prod>" + + "</deployment>" + ) + ); + + private final DeploymentSpec deploymentSpecNoServiceId = DeploymentSpec.fromXml( + new StringReader( + "<deployment>" + + " <prod>" + + " <region active='true'>us-east</region>" + + " <region active='true'>us-west</region>" + + " </prod>" + + "</deployment>" + ) + ); + + private final DeploymentSpec deploymentSpecOnlyOneNonCorpRegion = DeploymentSpec.fromXml( + new StringReader( + "<deployment>" + + " <prod global-service-id='nalle'>" + + " <region active='true'>us-east</region>" + + " <region active='true'>corp-us-west</region>" + + " </prod>" + + "</deployment>" + ) + ); + + private final DeploymentSpec deploymentSpecWithAdditionalCorpZone = DeploymentSpec.fromXml( + new StringReader( + "<deployment>" + + " <prod global-service-id='nalle'>" + + " <region active='true'>us-east</region>" + + " <region active='true'>corp-us-west</region>" + + " <region active='true'>us-west</region>" + + " </prod>" + + "</deployment>" + ) + ); + + private ControllerRotationRepository repository; + private ControllerRotationRepository repositoryWhitespaces; + + + @Before + public void setup_repository() { + repository = new ControllerRotationRepository(rotationsConfig, controllerDb, MetricReceiver.nullImplementation); + repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, MetricReceiver.nullImplementation); + controllerDb.assignRotation(new RotationId("foo-1"), applicationId); + } + + @Test + public void application_with_rotation_reused() { + Set<Rotation> rotations = repository.getOrAssignRotation(applicationId, deploymentSpec); + Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); + assertThat(rotations).containsOnly(assignedRotation); + } + + @Test + public void names_stripped() { + Set<Rotation> rotations = repositoryWhitespaces.getOrAssignRotation(applicationId, deploymentSpec); + Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); + assertThat(rotations).containsOnly(assignedRotation); + } + + @Test + public void application_without_rotation() { + ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default"); + Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpec); + Rotation assignedRotation = new Rotation(new RotationId("foo-2"), "foo-2.com"); + assertThat(rotations).containsOnly(assignedRotation); + } + + @Test + public void application_without_rotation_but_none_left() { + application_without_rotation(); // run this test to assign last rotation + ApplicationId third = ApplicationId.from("thirdtenant", "thirdapplication", "default"); + + thrown.expect(RuntimeException.class); + thrown.expectMessage("no rotations available"); + + repository.getOrAssignRotation(third, deploymentSpec); + } + + @Test + public void application_without_rotation_but_does_not_qualify() { + ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default"); + + thrown.expect(RuntimeException.class); + thrown.expectMessage("less than 2 prod zones are defined"); + + repository.getOrAssignRotation(other, deploymentSpecOneRegion); + } + + @Test + public void application_with_rotation_but_does_not_qualify() { + Set<Rotation> rotations = repository.getOrAssignRotation(applicationId, deploymentSpecOneRegion); + Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); + assertThat(rotations).containsOnly(assignedRotation); + } + + @Test + public void application_with_rotation_is_listed() { + repository.getOrAssignRotation(applicationId, deploymentSpec); + Set<URI> uris = repository.getRotationUris(applicationId); + assertThat(uris).isEqualTo( + Collections.singleton(URI.create("http://tumblr-search.msbe.global.vespa.yahooapis.com:4080/")) + ); + } + + @Test + public void application_without_rotation_is_empty() { + ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default"); + Set<URI> uris = repository.getRotationUris(other); + assertThat(uris).isEmpty(); + } + + @Test + public void application_without_serviceid_and_two_regions() { + ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default"); + Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpecNoServiceId); + assertThat(rotations).isEmpty(); + } + + @Test + public void application_with_only_one_non_corp_region() { + ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default"); + + thrown.expect(RuntimeException.class); + thrown.expectMessage("less than 2 prod zones are defined"); + + repository.getOrAssignRotation(other, deploymentSpecOnlyOneNonCorpRegion); + } + + @Test + public void application_with_corp_region_and_two_non_corp_region() { + ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default"); + Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpecWithAdditionalCorpZone); + assertThat(rotations).containsOnly(new Rotation(new RotationId("foo-2"), "foo-2.com")); + } + +} diff --git a/controller-server/src/test/resources/chef_output.json b/controller-server/src/test/resources/chef_output.json new file mode 100644 index 00000000000..257065f7b5b --- /dev/null +++ b/controller-server/src/test/resources/chef_output.json @@ -0,0 +1,34 @@ +{ + "total": 1, + "start": 0, + "rows": [ + { + "url": "https://chef-server.test/organizations/vespa/nodes/fake-node.test", + "data": { + "fqdn": "fake-node.test", + "ohai_time": 1475497186.68962, + "tenant": "ciintegrationtests", + "application": "restart", + "instance": "default", + "zone": "cd_cd-us-east-1_prod", + "system": "cd", + "environment": "prod", + "region": "cd-us-east-1" + } + }, + { + "url": "https://chef-server.test/organizations/vespa/nodes/fake-node2.test", + "data": { + "fqdn": "fake-node2.test", + "ohai_time": 1475497186.68962, + "tenant": null, + "application": null, + "instance": null, + "zone": null, + "system": null, + "environment": null, + "region": null + } + } + ] +} diff --git a/controller-server/src/test/resources/job-grandparent.json b/controller-server/src/test/resources/job-grandparent.json new file mode 100644 index 00000000000..63602bed146 --- /dev/null +++ b/controller-server/src/test/resources/job-grandparent.json @@ -0,0 +1,4 @@ +{ + "duration": 720000, + "causes": [] +} diff --git a/controller-server/src/test/resources/job-parent.json b/controller-server/src/test/resources/job-parent.json new file mode 100644 index 00000000000..88d50de394f --- /dev/null +++ b/controller-server/src/test/resources/job-parent.json @@ -0,0 +1,9 @@ +{ + "duration": 1200000, + "causes": [ + { + "upstreamBuild": 231, + "upstreamProject": "3-v3-job-grandparent" + } + ] +} diff --git a/controller-server/src/test/resources/job.json b/controller-server/src/test/resources/job.json new file mode 100644 index 00000000000..845566867b7 --- /dev/null +++ b/controller-server/src/test/resources/job.json @@ -0,0 +1,9 @@ +{ + "duration": 600000, + "causes": [ + { + "upstreamBuild": 123, + "upstreamProject": "2-v3-job-parent" + } + ] +} diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp index 7cad7356c8e..d8920b0577b 100644 --- a/documentapi/src/tests/messagebus/messagebus_test.cpp +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -81,7 +81,7 @@ void Test::testMessage() { new document::DocumentUpdate(*testdoc_type, document::DocumentId(document::DocIdString( "testdoc", "testme2"))))); - EXPECT_TRUE(!(upd1.getDocumentUpdate()->getId() == upd2.getDocumentUpdate()->getId())); + EXPECT_TRUE(!(upd1.getDocumentUpdate().getId() == upd2.getDocumentUpdate().getId())); DocumentMessage& msg2 = static_cast<DocumentMessage&>(upd2); EXPECT_TRUE(msg2.getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT); diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 8c20ef77201..964e4e12288 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -444,8 +444,8 @@ Messages50Test::testPutDocumentMessage() mbus::Routable::UP obj = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang); if (EXPECT_TRUE(obj.get() != NULL)) { PutDocumentMessage &ref = static_cast<PutDocumentMessage&>(*obj); - EXPECT_TRUE(ref.getDocument()->getType().getName() == "testdoc"); - EXPECT_TRUE(ref.getDocument()->getId().toString() == "doc:scheme:"); + EXPECT_TRUE(ref.getDocument().getType().getName() == "testdoc"); + EXPECT_TRUE(ref.getDocument().getId().toString() == "doc:scheme:"); EXPECT_EQUAL(666u, ref.getTimestamp()); EXPECT_EQUAL(37u, ref.getApproxSize()); } @@ -737,7 +737,7 @@ Messages50Test::testUpdateDocumentMessage() mbus::Routable::UP obj = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); if (EXPECT_TRUE(obj.get() != NULL)) { UpdateDocumentMessage &ref = static_cast<UpdateDocumentMessage&>(*obj); - EXPECT_EQUAL(*upd, *ref.getDocumentUpdate()); + EXPECT_EQUAL(*upd, ref.getDocumentUpdate()); EXPECT_EQUAL(666u, ref.getOldTimestamp()); EXPECT_EQUAL(777u, ref.getNewTimestamp()); EXPECT_EQUAL(85u, ref.getApproxSize()); @@ -1047,8 +1047,8 @@ Messages50Test::testGetDocumentReply() if (EXPECT_TRUE(obj.get() != NULL)) { GetDocumentReply &ref = static_cast<GetDocumentReply&>(*obj); - EXPECT_EQUAL(string("testdoc"), ref.getDocument()->getType().getName()); - EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument()->getId().toString()); + EXPECT_EQUAL(string("testdoc"), ref.getDocument().getType().getName()); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument().getId().toString()); } } return true; diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp index d6394012688..33eb5134dce 100644 --- a/documentapi/src/tests/messages/messages52test.cpp +++ b/documentapi/src/tests/messages/messages52test.cpp @@ -52,8 +52,8 @@ Messages52Test::testPutDocumentMessage() if (EXPECT_TRUE(routableUp.get() != nullptr)) { auto & deserializedMsg = static_cast<PutDocumentMessage &>(*routableUp); - EXPECT_EQUAL(msg.getDocument()->getType().getName(), deserializedMsg.getDocument()->getType().getName()); - EXPECT_EQUAL(msg.getDocument()->getId().toString(), deserializedMsg.getDocument()->getId().toString()); + EXPECT_EQUAL(msg.getDocument().getType().getName(), deserializedMsg.getDocument().getType().getName()); + EXPECT_EQUAL(msg.getDocument().getId().toString(), deserializedMsg.getDocument().getId().toString()); EXPECT_EQUAL(msg.getTimestamp(), deserializedMsg.getTimestamp()); EXPECT_EQUAL(67u, deserializedMsg.getApproxSize()); EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); @@ -90,8 +90,7 @@ Messages52Test::testUpdateDocumentMessage() const DocumentTypeRepo & repo = getTypeRepo(); const document::DocumentType & docType = *repo.getDocumentType("testdoc"); - document::DocumentUpdate::SP docUpdate(new document::DocumentUpdate(docType, - document::DocumentId("doc:scheme:"))); + auto docUpdate = std::make_shared<document::DocumentUpdate>(docType, document::DocumentId("doc:scheme:")); docUpdate->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); @@ -108,7 +107,7 @@ Messages52Test::testUpdateDocumentMessage() if (EXPECT_TRUE(routableUp.get() != nullptr)) { auto & deserializedMsg = static_cast<UpdateDocumentMessage &>(*routableUp); - EXPECT_EQUAL(*msg.getDocumentUpdate(), *deserializedMsg.getDocumentUpdate()); + EXPECT_EQUAL(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); EXPECT_EQUAL(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); EXPECT_EQUAL(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); EXPECT_EQUAL(115u, deserializedMsg.getApproxSize()); diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 0eb08f1e632..3629604aeea 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -738,7 +738,7 @@ Test::multipleGetRepliesAreMergedToFoundDocument() doc.reset(new Document(*_docType, DocumentId("doc:scheme:yarn"))); doc->setLastModified(123456ULL); } - mbus::Reply::UP reply(new GetDocumentReply(doc)); + mbus::Reply::UP reply(new GetDocumentReply(std::move(doc))); selected[i]->handleReply(std::move(reply)); } mbus::Reply::UP reply = frame.getReceptor().getReply(600); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp index 41431dc305e..c7422a529a3 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp @@ -2,6 +2,7 @@ #include "getdocumentreply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/document/fieldvalue/document.h> namespace documentapi { @@ -15,30 +16,18 @@ GetDocumentReply::~GetDocumentReply() {} GetDocumentReply::GetDocumentReply(document::Document::SP document) : DocumentAcceptedReply(DocumentProtocol::REPLY_GETDOCUMENT), - _document(document), + _document(std::move(document)), _lastModified(0) { - if (_document.get()) { + if (_document) { _lastModified = _document->getLastModified(); } } -document::Document::SP -GetDocumentReply::getDocument() -{ - return _document; -} - -std::shared_ptr<const document::Document> -GetDocumentReply::getDocument() const -{ - return _document; -} - void GetDocumentReply::setDocument(document::Document::SP document) { - _document = document; + _document = std::move(document); if (document.get()) { _lastModified = document->getLastModified(); } else { @@ -46,10 +35,4 @@ GetDocumentReply::setDocument(document::Document::SP document) } } -void -GetDocumentReply::setLastModified(uint64_t lastModified) -{ - _lastModified = lastModified; -} - } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h index 04859af51ea..2cff36325ab 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h @@ -2,14 +2,16 @@ #pragma once #include "documentacceptedreply.h" -#include <vespa/document/fieldvalue/document.h> + +namespace document { class Document; } namespace documentapi { class GetDocumentReply : public DocumentAcceptedReply { private: - document::Document::SP _document; - uint64_t _lastModified; + using DocumentSP = std::shared_ptr<document::Document>; + DocumentSP _document; + uint64_t _lastModified; public: /** @@ -29,28 +31,22 @@ public: * * @param document The document requested. */ - GetDocumentReply(document::Document::SP document); - - /** - * Returns the document retrieved. - * - * @return The document. - */ - document::Document::SP getDocument(); + GetDocumentReply(DocumentSP document); /** * Returns the document retrieved. * * @return The document. */ - std::shared_ptr<const document::Document> getDocument() const; + const document::Document & getDocument() const { return *_document; } + bool hasDocument() const { return _document.get() != nullptr; } /** * Sets the document retrieved. * * @param document The document. */ - void setDocument(document::Document::SP document); + void setDocument(DocumentSP document); /** * Returns the date the document was last modified. @@ -64,7 +60,7 @@ public: * * @param lastModified The date. */ - void setLastModified(uint64_t lastModified); + void setLastModified(uint64_t lastModified) { _lastModified = lastModified; } string toString() const override { return "getdocumentreply"; } }; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp index efd2e405267..6753d269ad6 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp @@ -2,6 +2,7 @@ #include "putdocumentmessage.h" #include "writedocumentreply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/exceptions.h> namespace documentapi { @@ -17,7 +18,7 @@ PutDocumentMessage::PutDocumentMessage(document::Document::SP document) : _document(), _time(0) { - setDocument(document); + setDocument(std::move(document)); } PutDocumentMessage::~PutDocumentMessage() {} @@ -46,25 +47,13 @@ PutDocumentMessage::getType() const return DocumentProtocol::MESSAGE_PUTDOCUMENT; } -document::Document::SP -PutDocumentMessage::getDocument() -{ - return _document; -} - -std::shared_ptr<const document::Document> -PutDocumentMessage::getDocument() const -{ - return _document; -} - void PutDocumentMessage::setDocument(document::Document::SP document) { - if (document.get() == NULL) { + if ( ! document ) { throw vespalib::IllegalArgumentException("Document can not be null.", VESPA_STRLOC); } - _document = document; + _document = std::move(document); } } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h index e4b43aaaf37..9bc1c088dfa 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h @@ -2,25 +2,22 @@ #pragma once #include "testandsetmessage.h" -#include <vespa/document/fieldvalue/document.h> +namespace document { class Document; } namespace documentapi { class PutDocumentMessage : public TestAndSetMessage { private: - document::Document::SP _document; - uint64_t _time; + using DocumentSP = std::shared_ptr<document::Document>; + DocumentSP _document; + uint64_t _time; protected: - // Implements DocumentMessage. DocumentReply::UP doCreateReply() const override; public: - /** - * Convenience typedef. - */ - typedef std::unique_ptr<PutDocumentMessage> UP; - typedef std::shared_ptr<PutDocumentMessage> SP; + using UP = std::unique_ptr<PutDocumentMessage>; + using SP = std::shared_ptr<PutDocumentMessage>; /** * Constructs a new document message for deserialization. @@ -32,7 +29,7 @@ public: * * @param document The document to put. */ - PutDocumentMessage(document::Document::SP document); + PutDocumentMessage(DocumentSP document); ~PutDocumentMessage(); /** @@ -40,21 +37,15 @@ public: * * @return The document. */ - document::Document::SP getDocument(); - - /** - * Returns the document to put. - * - * @return The document. - */ - std::shared_ptr<const document::Document> getDocument() const; + const DocumentSP & getDocumentSP() const { return _document; } + const document::Document & getDocument() const { return *_document; } /** * Sets the document to put. * * @param document The document to set. */ - void setDocument(document::Document::SP document); + void setDocument(DocumentSP document); /** * Returns the timestamp of the document to put. @@ -69,13 +60,9 @@ public: * @param time The timestamp to set. */ void setTimestamp(uint64_t time) { _time = time; } - bool hasSequenceId() const override; - uint64_t getSequenceId() const override; - uint32_t getType() const override; - string toString() const override { return "putdocumentmessage"; } }; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp index ef8a2b74298..db5dafce271 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp @@ -3,6 +3,7 @@ #include "updatedocumentmessage.h" #include "updatedocumentreply.h" #include <vespa/documentapi/messagebus/documentprotocol.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/util/exceptions.h> namespace documentapi { @@ -20,7 +21,7 @@ UpdateDocumentMessage::UpdateDocumentMessage(document::DocumentUpdate::SP docume _oldTime(0), _newTime(0) { - setDocumentUpdate(documentUpdate); + setDocumentUpdate(std::move(documentUpdate)); } UpdateDocumentMessage::~UpdateDocumentMessage() {} @@ -49,25 +50,13 @@ UpdateDocumentMessage::getType() const return DocumentProtocol::MESSAGE_UPDATEDOCUMENT; } -document::DocumentUpdate::SP -UpdateDocumentMessage::getDocumentUpdate() -{ - return _documentUpdate; -} - -std::shared_ptr<const document::DocumentUpdate> -UpdateDocumentMessage::getDocumentUpdate() const -{ - return _documentUpdate; -} - void UpdateDocumentMessage::setDocumentUpdate(document::DocumentUpdate::SP documentUpdate) { - if (documentUpdate.get() == NULL) { + if ( ! documentUpdate) { throw vespalib::IllegalArgumentException("Document update can not be null.", VESPA_STRLOC); } - _documentUpdate = documentUpdate; + _documentUpdate = std::move(documentUpdate); } } diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h index 25991191eeb..3a320960515 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h @@ -2,15 +2,17 @@ #pragma once #include "testandsetmessage.h" -#include <vespa/document/update/documentupdate.h> + +namespace document { class DocumentUpdate; } namespace documentapi { class UpdateDocumentMessage : public TestAndSetMessage { private: - document::DocumentUpdate::SP _documentUpdate; - uint64_t _oldTime; - uint64_t _newTime; + using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>; + DocumentUpdateSP _documentUpdate; + uint64_t _oldTime; + uint64_t _newTime; protected: DocumentReply::UP doCreateReply() const override; @@ -33,28 +35,21 @@ public: * * @param documentUpdate The document update to perform. */ - UpdateDocumentMessage(document::DocumentUpdate::SP documentUpdate); - - /** - * Returns the document update to perform. - * - * @return The update. - */ - document::DocumentUpdate::SP getDocumentUpdate(); + UpdateDocumentMessage(DocumentUpdateSP documentUpdate); /** * Returns the document update to perform. * * @return The update. */ - std::shared_ptr<const document::DocumentUpdate> getDocumentUpdate() const; - + const DocumentUpdateSP & getDocumentUpdateSP() const { return _documentUpdate; } + const document::DocumentUpdate & getDocumentUpdate() const { return *_documentUpdate; } /** * Sets the document update to perform. * * @param documentUpdate The document update to set. */ - void setDocumentUpdate(document::DocumentUpdate::SP documentUpdate); + void setDocumentUpdate(DocumentUpdateSP documentUpdate); /** * Returns the timestamp required for this update to be applied. diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index 011b54305bb..6756f694267 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -124,12 +124,10 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespali const mbus::Message &msg = context.getMessage(); switch(msg.getType()) { case DocumentProtocol::MESSAGE_PUTDOCUMENT: - return it->second->contains(*static_cast<const PutDocumentMessage&>(msg).getDocument()) == - Result::True; + return it->second->contains(static_cast<const PutDocumentMessage&>(msg).getDocument()) == Result::True; case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: - return it->second->contains(*static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) != - Result::False; + return it->second->contains(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) != Result::False; case DocumentProtocol::MESSAGE_MULTIOPERATION: { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp index a6b1e200cd4..38610aca551 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp @@ -53,20 +53,20 @@ SearchColumnPolicy::select(mbus::RoutingContext &context) const mbus::Message &msg = context.getMessage(); switch(msg.getType()) { case DocumentProtocol::MESSAGE_PUTDOCUMENT: - id = &static_cast<const PutDocumentMessage&>(msg).getDocument()->getId(); + id = &static_cast<const PutDocumentMessage&>(msg).getDocument().getId(); break; case DocumentProtocol::MESSAGE_GETDOCUMENT: - id = &static_cast<const GetDocumentMessage&>(msg).getDocumentId(); + id = &static_cast<const GetDocumentMessage&>(msg).getDocumentId(); break; case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: - id = &static_cast<const RemoveDocumentMessage&>(msg).getDocumentId(); + id = &static_cast<const RemoveDocumentMessage&>(msg).getDocumentId(); break; case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: - id = &static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()->getId(); - break; + id = &static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate().getId(); + break; case DocumentProtocol::MESSAGE_MULTIOPERATION: bucketId = (static_cast<const MultiOperationMessage&>(msg)).getBucketId(); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index cace5b6576a..b7b451e8ddf 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -119,7 +119,7 @@ StoragePolicy::doSelect(mbus::RoutingContext &context) document::BucketId id; switch(msg.getType()) { case DocumentProtocol::MESSAGE_PUTDOCUMENT: - id = _bucketIdFactory.getBucketId(static_cast<const PutDocumentMessage&>(msg).getDocument()->getId()); + id = _bucketIdFactory.getBucketId(static_cast<const PutDocumentMessage&>(msg).getDocument().getId()); break; case DocumentProtocol::MESSAGE_GETDOCUMENT: @@ -131,7 +131,7 @@ StoragePolicy::doSelect(mbus::RoutingContext &context) break; case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: - id = _bucketIdFactory.getBucketId(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()->getId()); + id = _bucketIdFactory.getBucketId(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate().getId()); break; case DocumentProtocol::MESSAGE_MULTIOPERATION: diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index e1b6035d5e0..504b3fcdf4f 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -567,14 +567,15 @@ RoutableFactories50::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf GetDocumentReply &reply = static_cast<GetDocumentReply&>(*ret); bool hasDocument = decodeBoolean(buf); - document::Document::SP document; + document::Document * document = nullptr; if (hasDocument) { - document.reset(new document::Document(_repo, buf)); - reply.setDocument(document); + auto doc = std::make_shared<document::Document>(_repo, buf); + document = doc.get(); + reply.setDocument(std::move(doc)); } int64_t lastModified = decodeLong(buf); reply.setLastModified(lastModified); - if (document.get()) { + if (hasDocument) { document->setLastModified(lastModified); } @@ -586,10 +587,10 @@ RoutableFactories50::GetDocumentReplyFactory::doEncode(const DocumentReply &obj, { const GetDocumentReply &reply = static_cast<const GetDocumentReply&>(obj); - buf.putByte(reply.getDocument().get() == NULL ? 0 : 1); - if (reply.getDocument().get() != NULL) { + buf.putByte(reply.hasDocument() ? 1 : 0); + if (reply.hasDocument()) { nbostream stream; - reply.getDocument()->serialize(stream); + reply.getDocument().serialize(stream); buf.putBytes(stream.peek(), stream.size()); } buf.putLong(reply.getLastModified()); @@ -693,7 +694,7 @@ RoutableFactories50::PutDocumentMessageFactory::doEncode(const DocumentMessage & auto & msg = static_cast<const PutDocumentMessage &>(obj); nbostream stream; - msg.getDocument()->serialize(stream); + msg.getDocument().serialize(stream); buf.putBytes(stream.peek(), stream.size()); buf.putLong(static_cast<int64_t>(msg.getTimestamp())); @@ -950,7 +951,7 @@ RoutableFactories50::UpdateDocumentMessageFactory::doEncode(const DocumentMessag const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); vespalib::nbostream stream; - msg.getDocumentUpdate()->serializeHEAD(stream); + msg.getDocumentUpdate().serializeHEAD(stream); buf.putBytes(stream.peek(), stream.size()); buf.putLong((int64_t)msg.getOldTimestamp()); buf.putLong((int64_t)msg.getNewTimestamp()); diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java index 51b6d285819..921c10a1821 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java @@ -8,7 +8,9 @@ import org.osgi.framework.ServiceReference; import org.osgi.service.log.LogEntry; import org.osgi.service.log.LogReaderService; -import java.util.Enumeration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -20,6 +22,7 @@ import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author bjorncs */ public class OsgiLogServiceIntegrationTest { @@ -35,28 +38,26 @@ public class OsgiLogServiceIntegrationTest { BundleContext ctx = driver.osgiFramework().bundleContext(); ServiceReference<?> ref = ctx.getServiceReference(LogReaderService.class.getName()); LogReaderService reader = (LogReaderService)ctx.getService(ref); - Enumeration<LogEntry> log = (Enumeration<LogEntry>)reader.getLog(); + ArrayList<LogEntry> logEntries = Collections.list(reader.getLog()); + assertTrue(logEntries.size() >= 4); - assertEntry(Level.INFO, "[jdk14] hello world", null, now, log); - assertEntry(Level.INFO, "[slf4j] hello world", null, now, log); - assertEntry(Level.INFO, "[log4j] hello world", null, now, log); - assertEntry(Level.INFO, "[jcl] hello world", null, now, log); + assertLogContainsEntry("[jdk14] hello world", logEntries, now); + assertLogContainsEntry("[slf4j] hello world", logEntries, now); + assertLogContainsEntry("[log4j] hello world", logEntries, now); + assertLogContainsEntry("[jcl] hello world", logEntries, now); assertTrue(driver.close()); } - private static void assertEntry(Level expectedLevel, String expectedMessage, Throwable expectedException, - long expectedTimeGE, Enumeration<LogEntry> log) + private static void assertLogContainsEntry(String expectedMessage, List<LogEntry> logEntries, long expectedTimeGE) { - assertTrue(log.hasMoreElements()); - LogEntry entry = log.nextElement(); - assertNotNull(entry); - System.err.println("log entry: "+entry.getMessage()+" bundle="+entry.getBundle()); - assertEquals(expectedMessage, entry.getMessage()); + LogEntry entry = logEntries.stream().filter(e -> e.getMessage().equals(expectedMessage)).findFirst() + .orElseThrow(() -> new AssertionError("Could not find log entry with messsage: " + expectedMessage)); + assertNull(entry.getBundle()); assertNotNull(entry.getServiceReference()); - assertEquals(OsgiLogHandler.toServiceLevel(expectedLevel), entry.getLevel()); - assertEquals(expectedException, entry.getException()); + assertEquals(OsgiLogHandler.toServiceLevel(Level.INFO), entry.getLevel()); + assertNull(entry.getException()); assertTrue(expectedTimeGE <= entry.getTime()); } } diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 20edbd084e5..75c5fc3f6c5 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -395,4 +395,3 @@ RPCNetwork::getMirror() const } } // namespace mbus - diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index c8f64a65415..8f7317b28eb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -112,7 +112,7 @@ public class History { public enum Type { // State move events - readied, reserved, activated, deactivated, deallocated, parked, + provisioned(false), readied, reserved, activated, deactivated, deallocated, parked, // The active node was retired retired, // The active node went down according to the service monitor diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 8056bd787db..9393dc5ead4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -257,6 +257,7 @@ public class NodeSerializer { /** Returns the event type, or null if this event type should be ignored */ private History.Event.Type eventTypeFromString(String eventTypeString) { switch (eventTypeString) { + case "provisioned" : return History.Event.Type.provisioned; case "readied" : return History.Event.Type.readied; case "reserved" : return History.Event.Type.reserved; case "activated" : return History.Event.Type.activated; @@ -273,6 +274,7 @@ public class NodeSerializer { } private String toString(History.Event.Type nodeEventType) { switch (nodeEventType) { + case provisioned : return "provisioned"; case readied : return "readied"; case reserved : return "reserved"; case activated : return "activated"; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java index 1b19b57317e..f91b4863eeb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java @@ -117,6 +117,10 @@ public class SerializationTest { " \"flavor\" : \"large\",\n" + " \"history\" : [\n" + " {\n" + + " \"type\" : \"provisioned\",\n" + + " \"at\" : 1444391401389\n" + + " },\n" + + " {\n" + " \"type\" : \"reserved\",\n" + " \"at\" : 1444391402611\n" + " }\n" + @@ -143,6 +147,8 @@ public class SerializationTest { assertEquals(2, node.status().reboot().current()); assertEquals(3, node.allocation().get().restartGeneration().wanted()); assertEquals(4, node.allocation().get().restartGeneration().current()); + assertEquals(Arrays.asList(History.Event.Type.provisioned, History.Event.Type.reserved), + node.history().events().stream().map(History.Event::type).collect(Collectors.toList())); assertTrue(node.allocation().get().isRemovable()); assertEquals(NodeType.tenant, node.type()); } @@ -941,6 +941,8 @@ <module>container-search-and-docproc</module> <module>container-search</module> <module>container-test-jars</module> + <module>controller-api</module> + <module>controller-server</module> <module>defaults</module> <module>docker-api</module> <module>docproc</module> diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index c38811e1962..d942049192a 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -4,13 +4,9 @@ #include <vespa/config/print/fileconfigwriter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/document.h> -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/documentapi/documentapi.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/messagebus/destinationsession.h> -#include <vespa/messagebus/imessagehandler.h> -#include <vespa/messagebus/iprotocol.h> -#include <vespa/messagebus/message.h> #include <vespa/messagebus/protocolset.h> #include <vespa/messagebus/rpcmessagebus.h> #include <vespa/vespalib/io/fileutil.h> @@ -56,7 +52,7 @@ private: OutputFile &_dat; size_t _numDocs; - void handleDocumentPut(document::Document::SP doc); + void handleDocumentPut(const document::Document::SP & doc); virtual void handleMessage(mbus::Message::UP message) override; public: @@ -66,9 +62,9 @@ public: }; void -FeedHandler::handleDocumentPut(document::Document::SP doc) +FeedHandler::handleDocumentPut(const document::Document::SP & doc) { - if (doc.get() != 0) { + if (doc) { vespalib::nbostream datStream(12345); vespalib::nbostream idxStream(12); doc->serialize(datStream); @@ -86,7 +82,7 @@ FeedHandler::handleMessage(mbus::Message::UP message) documentapi::DocumentMessage::UP msg((documentapi::DocumentMessage*)message.release()); switch (msg->getType()) { case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: - handleDocumentPut(((documentapi::PutDocumentMessage&)(*msg)).getDocument()); + handleDocumentPut(((documentapi::PutDocumentMessage&)(*msg)).getDocumentSP()); break; default: break; diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 4083a7e1194..f0a91a101eb 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -20,13 +20,13 @@ #include <vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h> #include <vespa/searchcore/proton/server/summaryadapter.h> #include <vespa/searchcore/proton/server/reconfig_params.h> +#include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/nosyncproxy.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> using namespace config; using namespace document; diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index 483e725927d..b9a04acb8da 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -3,7 +3,6 @@ #include <tests/proton/common/dummydbowner.h> #include <vespa/searchcore/proton/attribute/flushableattribute.h> #include <vespa/searchcore/proton/common/feedtoken.h> -#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/common/statusreport.h> #include <vespa/searchcore/proton/docsummary/summaryflushtarget.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h> @@ -12,11 +11,9 @@ #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> -#include <vespa/searchcore/proton/reference/document_db_reference_registry.h> #include <vespa/searchcore/proton/reference/i_document_db_reference.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/document_db_explorer.h> -#include <vespa/searchcore/proton/server/documentdb.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcorespi/index/indexflushtarget.h> diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h index 7ebc2c36985..92a9dc06fff 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h @@ -6,9 +6,7 @@ #include <vector> #include <memory> -namespace search { - -namespace grouping { +namespace search::grouping { /** * A Grouping Context contains all grouping expressions that should be evaluated @@ -115,6 +113,4 @@ public: bool needRanking() const; }; -} // namespace search::grouping -} // namespace search - +} diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index 7913043265a..48baba329ca 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -1,18 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "groupingmanager.h" +#include "groupingsession.h" +#include "groupingcontext.h" #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/expression/attributenode.h> -#include <vespa/searchcore/grouping/groupingsession.h> #include <vespa/log/log.h> LOG_SETUP(".groupingmanager"); -namespace search { -namespace grouping { +namespace search::grouping { -using search::aggregation::Grouping; -using search::attribute::IAttributeContext; +using aggregation::Grouping; +using attribute::IAttributeContext; //----------------------------------------------------------------------------- @@ -31,6 +31,10 @@ using search::expression::ConfigureStaticParams; using search::aggregation::Grouping; using search::aggregation::GroupingLevel; +bool GroupingManager::empty() const { + return _groupingContext.getGroupingList().empty(); +} + void GroupingManager::init(const IAttributeContext &attrCtx) { @@ -126,5 +130,4 @@ GroupingManager::convertToGlobalId(const search::IDocumentMetaStore &metaStore) } } -} // namespace search::grouping -} // namespace search +} diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h index 1a440bf9247..5793c8576e5 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h @@ -2,12 +2,16 @@ #pragma once #include <vespa/searchlib/common/idocumentmetastore.h> -#include <vespa/searchlib/aggregation/grouping.h> -#include <vespa/searchcore/grouping/groupingcontext.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> namespace search { + class RankedHit; + class BitVector; +} -namespace grouping { +namespace search::grouping { + +class GroupingContext; /** * Wrapper class used to handle actual grouping. All input data is @@ -37,14 +41,14 @@ public: /** * @return true if this manager is holding an empty grouping request. **/ - bool empty() const { return _groupingContext.getGroupingList().empty(); } + bool empty() const; /** * Initialize underlying context with attribute bindings. * * @param attrCtx attribute context **/ - void init(const search::attribute::IAttributeContext &attrCtx); + void init(const attribute::IAttributeContext &attrCtx); /** * Perform actual grouping on the given results. @@ -65,7 +69,7 @@ public: * @param binSize size of search result array * @param overflow The unranked hits. **/ - void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow); + void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const BitVector * overflow); /** * Merge another grouping context into the underlying context of @@ -89,9 +93,7 @@ public: * * @param metaStore the attribute used to map from lid to gid. **/ - void convertToGlobalId(const search::IDocumentMetaStore &metaStore); + void convertToGlobalId(const IDocumentMetaStore &metaStore); }; -} // namespace search::grouping -} // namespace search - +} diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp index 6407a29175e..110d032205e 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "groupingsession.h" +#include "groupingmanager.h" +#include "groupingcontext.h" + #include <vespa/log/log.h> LOG_SETUP(".groupingsession"); -namespace search { -namespace grouping { +namespace search::grouping { using search::aggregation::Group; using search::aggregation::Grouping; @@ -16,8 +18,8 @@ GroupingSession::GroupingSession(const SessionId &sessionId, GroupingContext & groupingContext, const IAttributeContext &attrCtx) : _sessionId(sessionId), - _mgrContext(groupingContext), - _groupingManager(_mgrContext), + _mgrContext(std::make_unique<GroupingContext>(groupingContext)), + _groupingManager(std::make_unique<GroupingManager>(*_mgrContext)), _timeOfDoom(groupingContext.getTimeOfDoom()) { init(groupingContext, attrCtx); @@ -46,31 +48,30 @@ GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext _groupingMap[gp->getId()] = gp; g = gp; } - _mgrContext.addGrouping(g); + _mgrContext->addGrouping(g); } - _groupingManager.init(attrCtx); + _groupingManager->init(attrCtx); } void GroupingSession::prepareThreadContextCreation(size_t num_threads) { if (num_threads > 1) { - _mgrContext.serialize(); // need copy of internal modified request + _mgrContext->serialize(); // need copy of internal modified request } } GroupingContext::UP GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx) { - GroupingContext::UP ctx(new GroupingContext(_mgrContext)); + GroupingContext::UP ctx(new GroupingContext(*_mgrContext)); if (thread_id == 0) { - GroupingContext::GroupingList &groupingList = _mgrContext.getGroupingList(); + GroupingContext::GroupingList &groupingList = _mgrContext->getGroupingList(); for (size_t i = 0; i < groupingList.size(); ++i) { ctx->addGrouping(groupingList[i]); } } else { - ctx->deserialize(_mgrContext.getResult().peek(), - _mgrContext.getResult().size()); + ctx->deserialize(_mgrContext->getResult().peek(), _mgrContext->getResult().size()); GroupingManager man(*ctx); man.init(attrCtx); } @@ -97,5 +98,4 @@ GroupingSession::continueExecution(GroupingContext & groupingContext) groupingContext.serialize(); } -} // namespace search::grouping -} // namespace search +} diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.h b/searchcore/src/vespa/searchcore/grouping/groupingsession.h index 62867289be7..95a5332b417 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingsession.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.h @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "groupingmanager.h" -#include "groupingcontext.h" #include "sessionid.h" -#include <vespa/searchlib/aggregation/grouping.h> +#include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/fastos/timestamp.h> +#include <vector> #include <map> +namespace search::aggregation { class Grouping; } +namespace search::grouping { -namespace search { - -namespace grouping { +class GroupingContext; +class GroupingManager; /** * A grouping session represents the execution of a grouping expression with one @@ -21,15 +22,15 @@ namespace grouping { class GroupingSession { private: - typedef std::shared_ptr<search::aggregation::Grouping> GroupingPtr; - typedef std::map<uint32_t, GroupingPtr> GroupingMap; - typedef std::vector<GroupingPtr> GroupingList; + using GroupingPtr = std::shared_ptr<aggregation::Grouping>; + using GroupingMap = std::map<uint32_t, GroupingPtr>; + using GroupingList = std::vector<GroupingPtr>; - SessionId _sessionId; - GroupingContext _mgrContext; - GroupingManager _groupingManager; - GroupingMap _groupingMap; - fastos::TimeStamp _timeOfDoom; + SessionId _sessionId; + std::unique_ptr<GroupingContext> _mgrContext; + std::unique_ptr<GroupingManager> _groupingManager; + GroupingMap _groupingMap; + fastos::TimeStamp _timeOfDoom; public: typedef std::unique_ptr<GroupingSession> UP; @@ -43,7 +44,7 @@ public: **/ GroupingSession(const SessionId & sessionId, GroupingContext & groupingContext, - const search::attribute::IAttributeContext &attrCtx); + const attribute::IAttributeContext &attrCtx); GroupingSession(const GroupingSession &) = delete; GroupingSession &operator=(const GroupingSession &) = delete; @@ -62,8 +63,7 @@ public: * @param groupingContext The current grouping context. * @param attrCtx attribute context. **/ - void init(GroupingContext & groupingContext, - const search::attribute::IAttributeContext &attrCtx); + void init(GroupingContext & groupingContext, const attribute::IAttributeContext &attrCtx); /** * This function is called to prepare for creation of individual @@ -85,13 +85,12 @@ public: * @param thread_id thread id * @param attrCtx attribute context. **/ - GroupingContext::UP createThreadContext(size_t thread_id, - const search::attribute::IAttributeContext &attrCtx); + std::unique_ptr<GroupingContext> createThreadContext(size_t thread_id, const attribute::IAttributeContext &attrCtx); /** * Return the GroupingManager to use when performing grouping. **/ - GroupingManager & getGroupingManager() { return _groupingManager; } + GroupingManager & getGroupingManager() { return *_groupingManager; } /** * Continue excuting a query given a context. @@ -112,6 +111,4 @@ public: fastos::TimeStamp getTimeOfDoom() const { return _timeOfDoom; } }; -} // namespace search::grouping -} // namespace search - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 4ece0c65593..1d5abcc2929 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -9,13 +9,13 @@ #include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/util/thread_bundle.h> #include <vespa/searchcore/grouping/groupingmanager.h> +#include <vespa/searchcore/grouping/groupingcontext.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_thread"); -namespace proton { -namespace matching { +namespace proton::matching { using search::queryeval::OptimizedAndNotForBlackListing; using search::queryeval::SearchIterator; @@ -402,5 +402,4 @@ MatchThread::run() mergeDirector.dualMerge(thread_id, *resultContext->result, resultContext->groupingSource); } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index 9287089c34e..cd01c330931 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -15,8 +15,7 @@ #include <vespa/searchlib/common/sortresults.h> #include <vespa/searchlib/queryeval/hitcollector.h> -namespace proton { -namespace matching { +namespace proton::matching { /** * Runs a single match thread and keeps track of local state. @@ -111,5 +110,4 @@ public: PartialResult::UP extract_result() { return std::move(resultContext->result); } }; -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 852176e4918..32775d7619a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -7,6 +7,7 @@ #include "match_params.h" #include "matcher.h" #include "sessionmanager.h" +#include <vespa/searchcore/grouping/groupingcontext.h> #include <vespa/searchlib/engine/errorcodes.h> #include <vespa/searchlib/engine/docsumrequest.h> #include <vespa/searchlib/engine/searchrequest.h> diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index 1e7acead748..e2c6affebe1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -3,6 +3,8 @@ #include "result_processor.h" #include "partial_result.h" #include "sessionmanager.h" +#include <vespa/searchcore/grouping/groupingmanager.h> +#include <vespa/searchcore/grouping/groupingcontext.h> #include <vespa/searchlib/common/docstamp.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/engine/searchreply.h> @@ -15,8 +17,7 @@ using search::grouping::GroupingSession; using search::grouping::GroupingContext; using search::grouping::SessionId; -namespace proton { -namespace matching { +namespace proton::matching { ResultProcessor::Result::Result(std::unique_ptr<search::engine::SearchReply> reply, size_t numFs4Hits) : _reply(std::move(reply)), @@ -158,5 +159,4 @@ ResultProcessor::makeReply(PartialResultUP full_result) return Result::UP(new Result(std::move(reply), numFs4Hits)); } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h index 0a9b88c066a..a181c1660b7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h @@ -16,8 +16,7 @@ namespace search { class IDocumentMetaStore; } -namespace proton { -namespace matching { +namespace proton::matching { class SessionManager; class PartialResult; @@ -107,6 +106,4 @@ public: std::unique_ptr<Result> makeReply(PartialResultUP full_result); }; -} // namespace proton::matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp index f6fd7f7cc9d..e456ea5b5a2 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "session_manager_explorer.h" +#include "sessionmanager.h" #include <vespa/vespalib/data/slime/slime.h> using vespalib::slime::Inserter; using vespalib::slime::Cursor; using vespalib::StateExplorer; -namespace proton { -namespace matching { +namespace proton::matching { namespace { @@ -59,5 +59,4 @@ SessionManagerExplorer::get_child(vespalib::stringref name) const return std::unique_ptr<StateExplorer>(nullptr); } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h index 31dabc5a887..b8acfba0342 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h @@ -2,11 +2,11 @@ #pragma once -#include "sessionmanager.h" #include <vespa/vespalib/net/state_explorer.h> -namespace proton { -namespace matching { +namespace proton::matching { + +class SessionManager; /** * Class used to explore the state of a session manager @@ -20,8 +20,7 @@ public: SessionManagerExplorer(const SessionManager &manager) : _manager(manager) {} virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; virtual std::vector<vespalib::string> get_children_names() const override; - virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; + virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override; }; -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 2156d853ac4..6773798d2d7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -21,7 +21,6 @@ #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> -#include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> #include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> #include <vespa/searchcore/proton/persistenceengine/bucket_guard.h> @@ -44,6 +43,8 @@ class IDocumentDBOwner; class MetricsWireService; class StatusReport; +namespace matching { class SessionManager; } + /** * The document database contains all the necessary structures required per * document type. It has an internal single-threaded Executor to process input @@ -110,7 +111,7 @@ private: ProtonConfig::Summary _protonSummaryCfg; ProtonConfig::Index _protonIndexCfg; ConfigStore::UP _config_store; - matching::SessionManager::SP _sessionManager; // TODO: This should not have to be a shared pointer. + std::shared_ptr<matching::SessionManager> _sessionManager; // TODO: This should not have to be a shared pointer. MetricsWireService &_metricsWireService; MetricsUpdateHook _metricsHook; vespalib::VarHolder<IFeedView::SP> _feedView; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index c4ff03de6a1..c89a63b95c0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -60,8 +60,8 @@ public: }; private: - typedef vespa::config::search::AttributesConfig AttributesConfig; - typedef FastAccessDocSubDBConfigurer Configurer; + using AttributesConfig = vespa::config::search::AttributesConfig; + using Configurer = FastAccessDocSubDBConfigurer; const bool _hasAttributes; const bool _fastAccessAttributesOnly; @@ -81,7 +81,8 @@ private: void initFeedView(const IAttributeWriter::SP &writer, const DocumentDBConfig &configSnapshot); protected: - typedef StoreOnlyDocSubDB Parent; + using Parent = StoreOnlyDocSubDB; + using SessionManagerSP = std::shared_ptr<matching::SessionManager>; const bool _addMetrics; MetricsWireService &_metricsWireService; @@ -110,7 +111,7 @@ public: void setup(const DocumentSubDbInitializerResult &initResult) override; void initViews(const DocumentDBConfig &configSnapshot, - const matching::SessionManager::SP &sessionManager) override; + const SessionManagerSP &sessionManager) override; IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp index 0f38e48a8e2..3162f9a1c45 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp @@ -33,7 +33,7 @@ using matching::SessionManager; MatchView::MatchView(const Matchers::SP &matchers, const IndexSearchable::SP &indexSearchable, const IAttributeManager::SP &attrMgr, - const SessionManager::SP &sessionMgr, + const SessionManagerSP &sessionMgr, const IDocumentMetaStoreContext::SP &metaStore, DocIdLimit &docIdLimit) : _matchers(matchers), @@ -44,6 +44,7 @@ MatchView::MatchView(const Matchers::SP &matchers, _docIdLimit(docIdLimit) { } +MatchView::~MatchView() { } Matcher::SP MatchView::getMatcher(const vespalib::string & rankProfile) const diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h index 5207bce9288..511048f536f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h @@ -8,17 +8,20 @@ #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> #include <vespa/searchcore/proton/matching/match_context.h> #include <vespa/searchcore/proton/matching/matcher.h> -#include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcorespi/index/indexsearchable.h> #include <vespa/searchlib/attribute/attributevector.h> namespace proton { +namespace matching { + class SessionManager; +} class MatchView { + using SessionManagerSP = std::shared_ptr<matching::SessionManager>; Matchers::SP _matchers; searchcorespi::IndexSearchable::SP _indexSearchable; IAttributeManager::SP _attrMgr; - matching::SessionManager::SP _sessionMgr; + SessionManagerSP _sessionMgr; IDocumentMetaStoreContext::SP _metaStore; DocIdLimit &_docIdLimit; @@ -34,14 +37,15 @@ public: MatchView(const Matchers::SP &matchers, const searchcorespi::IndexSearchable::SP &indexSearchable, const IAttributeManager::SP &attrMgr, - const matching::SessionManager::SP &sessionMgr, + const SessionManagerSP &sessionMgr, const IDocumentMetaStoreContext::SP &metaStore, DocIdLimit &docIdLimit); + ~MatchView(); const Matchers::SP & getMatchers() const { return _matchers; } const searchcorespi::IndexSearchable::SP & getIndexSearchable() const { return _indexSearchable; } const IAttributeManager::SP & getAttributeManager() const { return _attrMgr; } - const matching::SessionManager::SP & getSessionManager() const { return _sessionMgr; } + const SessionManagerSP & getSessionManager() const { return _sessionMgr; } const IDocumentMetaStoreContext::SP & getDocumentMetaStore() const { return _metaStore; } DocIdLimit & getDocIdLimit() const { return _docIdLimit; } @@ -62,4 +66,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 7bd6c1ef100..fa3ded56c97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -21,8 +21,6 @@ #include <vespa/searchcorespi/index/iindexmanager.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/varholder.h> -#include <memory> -#include <vector> namespace proton { @@ -126,7 +124,7 @@ public: void initViews(const DocumentDBConfig &configSnapshot, - const matching::SessionManager::SP &sessionManager) override; + const SessionManagerSP &sessionManager) override; IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.h b/searchcore/src/vespa/searchcore/proton/server/searchview.h index cb1cd7670bc..186d6154706 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchview.h @@ -11,6 +11,7 @@ namespace proton { class SearchView : public ISearchHandler { public: + using SessionManagerSP = std::shared_ptr<matching::SessionManager>; using IndexSearchable = searchcorespi::IndexSearchable; using InternalDocsumReply = std::pair<std::unique_ptr<DocsumReply>, bool>; typedef std::shared_ptr<SearchView> SP; @@ -25,7 +26,7 @@ public: const Matchers::SP & getMatchers() const { return _matchView->getMatchers(); } const IndexSearchable::SP & getIndexSearchable() const { return _matchView->getIndexSearchable(); } const IAttributeManager::SP & getAttributeManager() const { return _matchView->getAttributeManager(); } - const matching::SessionManager::SP & getSessionManager() const { return _matchView->getSessionManager(); } + const SessionManagerSP & getSessionManager() const { return _matchView->getSessionManager(); } const IDocumentMetaStoreContext::SP & getDocumentMetaStore() const { return _matchView->getDocumentMetaStore(); } DocIdLimit &getDocIdLimit() const { return _matchView->getDocIdLimit(); } matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const { return _matchView->getMatcherStats(rankProfile); } diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index f855f51af42..6406ce6d4a1 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -304,7 +304,7 @@ template <typename FixtureType> void verify_get_string_from_enum_is_mapped(FixtureType& f) { EnumHandle handle{}; ASSERT_TRUE(f.target_attr->findEnum("foo", handle)); - const char* from_enum = f.imported_attr->getStringFromEnum(handle); + const char* from_enum = f.get_imported_attr()->getStringFromEnum(handle); ASSERT_TRUE(from_enum != nullptr); EXPECT_EQUAL(vespalib::string("foo"), vespalib::string(from_enum)); } @@ -438,7 +438,7 @@ struct MockAttributeVector : NotImplementedAttribute { long _return_value{1234}; MockAttributeVector() - : NotImplementedAttribute("mock", Config(BasicType::INT32)) { + : NotImplementedAttribute("mock", Config(BasicType::STRING)) { } void set_received_args(DocId doc_id, void* ser_to, @@ -475,44 +475,65 @@ struct MockBlobConverter : common::BlobConverter { } }; -struct SerializeFixture : Fixture { +template <typename BaseFixture> +struct SerializeFixture : BaseFixture { std::shared_ptr<MockAttributeVector> mock_target; MockBlobConverter mock_converter; - SerializeFixture() - : Fixture(), - mock_target(std::make_shared<MockAttributeVector>()) - { - reset_with_new_target_attr(mock_target); + SerializeFixture() : mock_target(std::make_shared<MockAttributeVector>()) { + this->reset_with_new_target_attr(mock_target); + mock_target->setCommittedDocIdLimit(8); // Target LID of 7 is highest used by ref attribute. Limit is +1. } + ~SerializeFixture() override; }; -TEST_F("onSerializeForAscendingSort() is forwarded to target vector", SerializeFixture) { +template <typename BaseFixture> +SerializeFixture<BaseFixture>::~SerializeFixture() {} + +template <typename FixtureT> +void check_onSerializeForAscendingSort_is_forwarded_with_remapped_lid() { + FixtureT f; int dummy_tag; void* ser_to = &dummy_tag; EXPECT_EQUAL(f.mock_target->_return_value, - f.imported_attr->serializeForAscendingSort( - DocId(10), ser_to, 777, &f.mock_converter)); + f.get_imported_attr()->serializeForAscendingSort( + DocId(4), ser_to, 777, &f.mock_converter)); // child lid 4 -> parent lid 7 EXPECT_TRUE(f.mock_target->_ascending_called); - EXPECT_EQUAL(DocId(10), f.mock_target->_doc_id); + EXPECT_EQUAL(DocId(7), f.mock_target->_doc_id); EXPECT_EQUAL(ser_to, f.mock_target->_ser_to); EXPECT_EQUAL(777, f.mock_target->_available); EXPECT_EQUAL(&f.mock_converter, f.mock_target->_bc); } -TEST_F("onSerializeForDescendingSort() is forwarded to target vector", SerializeFixture) { +TEST("onSerializeForAscendingSort() is forwarded with remapped LID to target vector") { + TEST_DO(check_onSerializeForAscendingSort_is_forwarded_with_remapped_lid< + SerializeFixture<SingleStringAttrFixture>>()); + TEST_DO(check_onSerializeForAscendingSort_is_forwarded_with_remapped_lid< + SerializeFixture<ReadGuardSingleStringAttrFixture>>()); +} + +template <typename FixtureT> +void check_onSerializeForDescendingSort_is_forwarded_with_remapped_lid() { + FixtureT f; int dummy_tag; void* ser_to = &dummy_tag; EXPECT_EQUAL(f.mock_target->_return_value, - f.imported_attr->serializeForDescendingSort( - DocId(20), ser_to, 555, &f.mock_converter)); + f.get_imported_attr()->serializeForDescendingSort( + DocId(2), ser_to, 555, &f.mock_converter)); // child lid 2 -> parent lid 3 EXPECT_TRUE(f.mock_target->_descending_called); - EXPECT_EQUAL(DocId(20), f.mock_target->_doc_id); + EXPECT_EQUAL(DocId(3), f.mock_target->_doc_id); EXPECT_EQUAL(ser_to, f.mock_target->_ser_to); EXPECT_EQUAL(555, f.mock_target->_available); EXPECT_EQUAL(&f.mock_converter, f.mock_target->_bc); } +TEST("onSerializeForDescendingSort() is forwarded with remapped LID to target vector") { + TEST_DO(check_onSerializeForDescendingSort_is_forwarded_with_remapped_lid< + SerializeFixture<SingleStringAttrFixture>>()); + TEST_DO(check_onSerializeForDescendingSort_is_forwarded_with_remapped_lid< + SerializeFixture<ReadGuardSingleStringAttrFixture>>()); +} + } // attribute } // search diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index 5811ca9cc4d..c9e0757ee16 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -197,6 +197,7 @@ TEST_F("Non-strict iterator unpacks target match data for weighted set hit", Wse TEST_F("Strict iterator is marked as strict", Fixture) { auto ctx = f.create_context(word_term("5678")); + ctx->fetchPostings(true); TermFieldMatchData match; auto iter = f.create_strict_iterator(*ctx, match); @@ -218,11 +219,12 @@ struct SingleValueFixture : Fixture { TEST_F("Strict iterator seeks to first available hit LID", SingleValueFixture) { auto ctx = f.create_context(word_term("5678")); + ctx->fetchPostings(true); TermFieldMatchData match; auto iter = f.create_strict_iterator(*ctx, match); EXPECT_FALSE(iter->isAtEnd()); - EXPECT_EQUAL(iter->beginId(), iter->getDocId()); + EXPECT_EQUAL(DocId(3), iter->getDocId()); EXPECT_FALSE(iter->seek(DocId(1))); EXPECT_FALSE(iter->isAtEnd()); @@ -243,6 +245,7 @@ TEST_F("Strict iterator seeks to first available hit LID", SingleValueFixture) { TEST_F("Strict iterator unpacks target match data for single value hit", SingleValueFixture) { auto ctx = f.create_context(word_term("5678")); + ctx->fetchPostings(true); TermFieldMatchData match; auto iter = f.create_strict_iterator(*ctx, match); @@ -254,6 +257,7 @@ TEST_F("Strict iterator unpacks target match data for single value hit", SingleV TEST_F("Strict iterator unpacks target match data for array hit", ArrayValueFixture) { auto ctx = f.create_context(word_term("1234")); + ctx->fetchPostings(true); TermFieldMatchData match; auto iter = f.create_strict_iterator(*ctx, match); @@ -265,6 +269,7 @@ TEST_F("Strict iterator unpacks target match data for array hit", ArrayValueFixt TEST_F("Strict iterator unpacks target match data for weighted set hit", WsetValueFixture) { auto ctx = f.create_context(word_term("foo")); + ctx->fetchPostings(true); TermFieldMatchData match; auto iter = f.create_strict_iterator(*ctx, match); @@ -275,6 +280,7 @@ TEST_F("Strict iterator unpacks target match data for weighted set hit", WsetVal TEST_F("Strict iterator handles seek outside of LID space", ArrayValueFixture) { auto ctx = f.create_context(word_term("1234")); + ctx->fetchPostings(true); TermFieldMatchData match; auto iter = f.create_strict_iterator(*ctx, match); @@ -306,6 +312,7 @@ TEST_F("cmp(weight) performs GID mapping and forwards to target attribute", Wset TEST_F("Multiple iterators can be created from the same context", SingleValueFixture) { auto ctx = f.create_context(word_term("5678")); + ctx->fetchPostings(true); TermFieldMatchData match1; auto iter1 = f.create_strict_iterator(*ctx, match1); diff --git a/searchlib/src/vespa/searchlib/aggregation/group.h b/searchlib/src/vespa/searchlib/aggregation/group.h index b34bd0fc88a..c769b6c1d27 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.h +++ b/searchlib/src/vespa/searchlib/aggregation/group.h @@ -64,7 +64,7 @@ public: size_t operator() (const ResultNode & arg) const { return arg.hash(); } }; - typedef std::vector<GroupingLevel> GroupingLevelList; + using GroupingLevelList = std::vector<GroupingLevel>; private: diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp index b6ff442ae94..270f4d51788 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp @@ -135,14 +135,16 @@ long ImportedAttributeVector::onSerializeForAscendingSort(DocId doc, void *serTo, long available, const common::BlobConverter *bc) const { - return _target_attribute->serializeForAscendingSort(doc, serTo, available, bc); + return _target_attribute->serializeForAscendingSort( + _reference_attribute->getReferencedLid(doc), serTo, available, bc); } long ImportedAttributeVector::onSerializeForDescendingSort(DocId doc, void *serTo, long available, const common::BlobConverter *bc) const { - return _target_attribute->serializeForDescendingSort(doc, serTo, available, bc); + return _target_attribute->serializeForDescendingSort( + _reference_attribute->getReferencedLid(doc), serTo, available, bc); } namespace { diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index 85f8e980026..563834e6cdb 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -80,5 +80,19 @@ uint32_t ImportedAttributeVectorReadGuard::get(DocId docId, WeightedEnum *buffer return _target_attribute->get(getReferencedLid(docId), buffer, sz); } +long ImportedAttributeVectorReadGuard::onSerializeForAscendingSort(DocId doc, + void *serTo, + long available, + const common::BlobConverter *bc) const { + return _target_attribute->serializeForAscendingSort(getReferencedLid(doc), serTo, available, bc); +} + +long ImportedAttributeVectorReadGuard::onSerializeForDescendingSort(DocId doc, + void *serTo, + long available, + const common::BlobConverter *bc) const { + return _target_attribute->serializeForDescendingSort(getReferencedLid(doc), serTo, available, bc); +} + } } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index 81a3d24b6cf..f4db2b538d5 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -48,6 +48,11 @@ public: virtual uint32_t get(DocId docId, WeightedString *buffer, uint32_t sz) const override; virtual uint32_t get(DocId docId, WeightedConstChar *buffer, uint32_t sz) const override; virtual uint32_t get(DocId docId, WeightedEnum *buffer, uint32_t sz) const override; +protected: + virtual long onSerializeForAscendingSort(DocId doc, void * serTo, long available, + const common::BlobConverter * bc) const override; + virtual long onSerializeForDescendingSort(DocId doc, void * serTo, long available, + const common::BlobConverter * bc) const override; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index c431b956c1d..b0ac12ce8f9 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -7,6 +7,19 @@ #include <vespa/searchcommon/attribute/search_context_params.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/query/queryterm.h> +#include <vespa/searchlib/queryeval/emptysearch.h> +#include "dociditerator.h" + +using search::datastore::EntryRef; +using search::queryeval::EmptySearch; +using search::queryeval::SearchIterator; +using search::attribute::ReferenceAttribute; +using search::AttributeVector; + +using ReverseMappingRefs = ReferenceAttribute::ReverseMappingRefs; +using ReverseMapping = ReferenceAttribute::ReverseMapping; +using SearchContext = AttributeVector::SearchContext; + namespace search { namespace attribute { @@ -19,7 +32,9 @@ ImportedSearchContext::ImportedSearchContext( _reference_attribute(*_imported_attribute.getReferenceAttribute()), _target_attribute(*_imported_attribute.getTargetAttribute()), _target_search_context(_target_attribute.getSearch(std::move(term), params)), - _referencedLids(_reference_attribute.getReferencedLids()) + _referencedLids(_reference_attribute.getReferencedLids()), + _merger(_reference_attribute.getCommittedDocIdLimit()), + _fetchPostingsDone(false) { } @@ -32,6 +47,18 @@ unsigned int ImportedSearchContext::approximateHits() const { std::unique_ptr<queryeval::SearchIterator> ImportedSearchContext::createIterator(fef::TermFieldMatchData* matchData, bool strict) { + if (_merger.hasArray()) { + if (_merger.emptyArray()) { + return SearchIterator::UP(new EmptySearch()); + } else { + using Posting = btree::BTreeKeyData<uint32_t, int32_t>; + using DocIt = DocIdIterator<Posting>; + DocIt postings; + auto array = _merger.getArray(); + postings.set(&array[0], &array[array.size()]); + return std::make_unique<AttributePostingListIteratorT<DocIt>>(true, matchData, postings); + } + } if (!strict) { return std::make_unique<AttributeIteratorT<ImportedSearchContext>>(*this, matchData); } else { @@ -39,9 +66,112 @@ ImportedSearchContext::createIterator(fef::TermFieldMatchData* matchData, bool s } } +namespace { + +struct WeightedRef { + EntryRef revMapIdx; + int32_t weight; + + WeightedRef(EntryRef revMapIdx_, int32_t weight_) + : revMapIdx(revMapIdx_), + weight(weight_) + { + } +}; + +struct TargetResult { + std::vector<WeightedRef> weightedRefs; + size_t sizeSum; + + TargetResult() + : weightedRefs(), + sizeSum(0) + { + } +}; + +TargetResult +getTargetResult(ReverseMappingRefs reverseMappingRefs, + const ReverseMapping &reverseMapping, + SearchContext &target_search_context, + uint32_t committedDocIdLimit) +{ + TargetResult targetResult; + fef::TermFieldMatchData matchData; + auto targetItr = target_search_context.createIterator(&matchData, true); + uint32_t docIdLimit = reverseMappingRefs.size(); + if (docIdLimit > committedDocIdLimit) { + docIdLimit = committedDocIdLimit; + } + uint32_t lid = 1; + targetItr->initRange(1, docIdLimit); + while (lid < docIdLimit) { + if (targetItr->seek(lid)) { + EntryRef revMapIdx = reverseMappingRefs[lid]; + if (revMapIdx.valid()) { + uint32_t size = reverseMapping.frozenSize(revMapIdx); + targetResult.sizeSum += size; + targetItr->unpack(lid); + int32_t weight = matchData.getWeight(); + targetResult.weightedRefs.emplace_back(revMapIdx, weight); + } + ++lid; + } else { + ++lid; + uint32_t nextLid = targetItr->getDocId(); + if (nextLid > lid) { + lid = nextLid; + } + } + } + return targetResult; +} + +class ReverseMappingPostingList +{ + const ReverseMapping &_reverseMapping; + EntryRef _revMapIdx; + int32_t _weight; +public: + ReverseMappingPostingList(const ReverseMapping &reverseMapping, EntryRef revMapIdx, int32_t weight) + : _reverseMapping(reverseMapping), + _revMapIdx(revMapIdx), + _weight(weight) + { + } + ~ReverseMappingPostingList() { } + template <typename Func> + void foreach(Func func) const { + int32_t weight = _weight; + _reverseMapping.foreach_frozen_key(_revMapIdx, [func, weight](uint32_t lid) { func(lid, weight); }); + } +}; + +} + +void ImportedSearchContext::makeMergedPostings() +{ + uint32_t committedTargetDocIdLimit = _target_attribute.getCommittedDocIdLimit(); + std::atomic_thread_fence(std::memory_order_acquire); + TargetResult targetResult(getTargetResult(_reference_attribute.getReverseMappingRefs(), + _reference_attribute.getReverseMapping(), + *_target_search_context, + committedTargetDocIdLimit)); + _merger.reserveArray(targetResult.weightedRefs.size(), targetResult.sizeSum); + const auto &reverseMapping = _reference_attribute.getReverseMapping(); + for (const auto &weightedRef : targetResult.weightedRefs) { + _merger.addToArray(ReverseMappingPostingList(reverseMapping, weightedRef.revMapIdx, weightedRef.weight)); + } + _merger.merge(); +} + void ImportedSearchContext::fetchPostings(bool strict) { - (void)strict; - // Imported attributes do not have posting lists (at least not currently), so this is a no-op. + assert(!_fetchPostingsDone); + _fetchPostingsDone = true; + _target_search_context->fetchPostings(strict); + if (strict) { + makeMergedPostings(); + } } bool ImportedSearchContext::valid() const { diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index ce6642cd93f..9be4578fac0 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -5,6 +5,7 @@ #include "attributevector.h" #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/vespalib/util/arrayref.h> +#include <vespa/searchlib/attribute/posting_list_merger.h> #include <memory> namespace search { @@ -34,6 +35,10 @@ class ImportedSearchContext : public ISearchContext { const AttributeVector& _target_attribute; std::unique_ptr<AttributeVector::SearchContext> _target_search_context; ReferencedLids _referencedLids; + PostingListMerger<int32_t> _merger; + bool _fetchPostingsDone; + + void makeMergedPostings(); public: ImportedSearchContext(std::unique_ptr<QueryTermSimple> term, const SearchContextParams& params, diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index 53b70c1da1d..4ee277b8733 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -36,6 +36,7 @@ public: btree::BTreeDefaultTraits, btree::NoAggrCalc>; using ReferencedLids = ReferenceMappings::ReferencedLids; + using ReverseMappingRefs = ReferenceMappings::ReverseMappingRefs; private: ReferenceStore _store; ReferenceStoreIndices _indices; @@ -74,6 +75,9 @@ public: std::shared_ptr<IGidToLidMapperFactory> getGidToLidMapperFactory() const { return _gidToLidMapperFactory; } ReferencedLids getReferencedLids() const { return _referenceMappings.getReferencedLids(); } DocId getReferencedLid(DocId doc) const { return _referenceMappings.getReferencedLid(doc); } + ReverseMappingRefs getReverseMappingRefs() const { return _referenceMappings.getReverseMappingRefs(); } + const ReverseMapping &getReverseMapping() const { return _referenceMappings.getReverseMapping(); } + void notifyGidToLidChange(const GlobalId &gid, DocId referencedLid); void populateReferencedLids(); virtual void clearDocs(DocId lidLow, DocId lidLimit) override; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp index 03acf6f2167..4edd9d45e60 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp @@ -9,6 +9,7 @@ namespace search::attribute { ReferenceMappings::ReferenceMappings(GenerationHolder &genHolder) : _reverseMappingIndices(genHolder), + _referencedLidLimit(0), _reverseMapping(), _referencedLids(genHolder) { @@ -38,7 +39,6 @@ ReferenceMappings::syncForwardMapping(const Reference &entry) { referencedLids[lid] = referencedLid; }); } - void ReferenceMappings::syncReverseMappingIndices(const Reference &entry) { @@ -46,6 +46,10 @@ ReferenceMappings::syncReverseMappingIndices(const Reference &entry) if (referencedLid != 0u) { _reverseMappingIndices.ensure_size(referencedLid + 1); _reverseMappingIndices[referencedLid] = entry.revMapIdx(); + if (referencedLid >= _referencedLidLimit) { + std::atomic_thread_fence(std::memory_order_release); + _referencedLidLimit = referencedLid + 1; + } } } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h index 631a610f773..3190e1b5a83 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/common/rcuvector.h> +#include <atomic> namespace search::attribute { @@ -27,6 +28,8 @@ class ReferenceMappings // Vector containing references to trees of lids referencing given // referenced lid. ReverseMappingIndices _reverseMappingIndices; + // limit for referenced lid when accessing _reverseMappingIndices + uint32_t _referencedLidLimit; // Store of B-Trees, used to map from gid or referenced lid to // referencing lids. ReverseMapping _reverseMapping; @@ -38,6 +41,7 @@ class ReferenceMappings public: using ReferencedLids = vespalib::ConstArrayRef<uint32_t>; + using ReverseMappingRefs = vespalib::ConstArrayRef<EntryRef>; ReferenceMappings(GenerationHolder &genHolder); @@ -76,6 +80,12 @@ public: ReferencedLids getReferencedLids() const { return ReferencedLids(&_referencedLids[0], _referencedLids.size()); } uint32_t getReferencedLid(uint32_t doc) const { return _referencedLids[doc]; } + ReverseMappingRefs getReverseMappingRefs() const { + uint32_t referencedLidLimit = _referencedLidLimit; + std::atomic_thread_fence(std::memory_order_acquire); + return ReverseMappingRefs(&_reverseMappingIndices[0], referencedLidLimit); + } + const ReverseMapping &getReverseMapping() const { return _reverseMapping; } }; template <typename FunctionType> diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp index 0ff4de00177..bab047827ad 100644 --- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp @@ -15,6 +15,12 @@ EmptySearch::doUnpack(uint32_t) { } +EmptySearch::Trinary +EmptySearch::is_strict() const +{ + return Trinary::True; +} + EmptySearch::EmptySearch() : SearchIterator() { diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h index c03c533deb1..12d7430922c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h @@ -16,6 +16,7 @@ protected: SearchIterator::initRange(begin, end); setAtEnd(); } + virtual Trinary is_strict() const override; public: EmptySearch(); diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index a0553625c8c..3830d3b71cb 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -13,7 +13,6 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/config/subscription/configuri.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <climits> using document::DataType; using document::DocIdString; @@ -78,15 +77,12 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DocumentApiConverterTest); void DocumentApiConverterTest::testPut() { - Document::SP - doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); documentapi::PutDocumentMessage putmsg(doc); putmsg.setTimestamp(1234); - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(putmsg, _repo); - + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(putmsg, _repo); api::PutCommand* pc = dynamic_cast<api::PutCommand*>(cmd.get()); CPPUNIT_ASSERT(pc); @@ -100,26 +96,23 @@ void DocumentApiConverterTest::testPut() api::PutReply* pr = dynamic_cast<api::PutReply*>(rep.get()); CPPUNIT_ASSERT(pr); - std::unique_ptr<mbus::Message> mbusmsg = - _converter->toDocumentAPI(*pc, _repo); + std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(*pc, _repo); documentapi::PutDocumentMessage* mbusput = dynamic_cast<documentapi::PutDocumentMessage*>(mbusmsg.get()); CPPUNIT_ASSERT(mbusput); - CPPUNIT_ASSERT(mbusput->getDocument().get() == doc.get()); + CPPUNIT_ASSERT(mbusput->getDocumentSP().get() == doc.get()); CPPUNIT_ASSERT(mbusput->getTimestamp() == 1234); }; void DocumentApiConverterTest::testForwardedPut() { - Document::SP - doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); documentapi::PutDocumentMessage* putmsg = new documentapi::PutDocumentMessage(doc); std::unique_ptr<mbus::Reply> reply(((documentapi::DocumentMessage*)putmsg)->createReply()); reply->setMessage(std::unique_ptr<mbus::Message>(putmsg)); - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(*putmsg, _repo); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(*putmsg, _repo); ((storage::api::PutCommand*)cmd.get())->setTimestamp(1234); std::unique_ptr<storage::api::StorageReply> rep = cmd->makeReply(); @@ -132,8 +125,7 @@ void DocumentApiConverterTest::testForwardedPut() void DocumentApiConverterTest::testRemove() { documentapi::RemoveDocumentMessage removemsg(document::DocumentId(document::DocIdString("test", "test"))); - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(removemsg, _repo); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(removemsg, _repo); api::RemoveCommand* rc = dynamic_cast<api::RemoveCommand*>(cmd.get()); @@ -148,8 +140,7 @@ void DocumentApiConverterTest::testRemove() api::RemoveReply* pr = dynamic_cast<api::RemoveReply*>(rep.get()); CPPUNIT_ASSERT(pr); - std::unique_ptr<mbus::Message> mbusmsg = - _converter->toDocumentAPI(*rc, _repo); + std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(*rc, _repo); documentapi::RemoveDocumentMessage* mbusremove = dynamic_cast<documentapi::RemoveDocumentMessage*>(mbusmsg.get()); CPPUNIT_ASSERT(mbusremove); @@ -159,11 +150,9 @@ void DocumentApiConverterTest::testRemove() void DocumentApiConverterTest::testGet() { documentapi::GetDocumentMessage getmsg( - document::DocumentId(document::DocIdString("test", "test")), - "foo bar"); + document::DocumentId(document::DocIdString("test", "test")), "foo bar"); - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(getmsg, _repo); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(getmsg, _repo); api::GetCommand* rc = dynamic_cast<api::GetCommand*>(cmd.get()); @@ -174,17 +163,10 @@ void DocumentApiConverterTest::testGet() void DocumentApiConverterTest::testCreateVisitor() { - documentapi::CreateVisitorMessage cv( - "mylib", - "myinstance", - "control-dest", - "data-dest"); + documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); cv.setTimeRemaining(123456); - - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(cv, _repo); - + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get()); CPPUNIT_ASSERT(pc); @@ -197,17 +179,9 @@ void DocumentApiConverterTest::testCreateVisitor() void DocumentApiConverterTest::testCreateVisitorHighTimeout() { - documentapi::CreateVisitorMessage cv( - "mylib", - "myinstance", - "control-dest", - "data-dest"); - + documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); cv.setTimeRemaining((uint64_t)std::numeric_limits<uint32_t>::max() + 1); // Will be INT_MAX - - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(cv, _repo); - + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get()); CPPUNIT_ASSERT(pc); @@ -215,65 +189,40 @@ void DocumentApiConverterTest::testCreateVisitorHighTimeout() CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), pc->getControlDestination()); CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), pc->getDataDestination()); - CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), - pc->getTimeout()); + CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), pc->getTimeout()); } void DocumentApiConverterTest::testCreateVisitorReplyNotReady() { - documentapi::CreateVisitorMessage cv( - "mylib", - "myinstance", - "control-dest", - "data-dest"); - - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(cv, _repo); + documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); CPPUNIT_ASSERT(cmd.get()); api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd); - api::CreateVisitorReply cvr(cvc); cvr.setResult(api::ReturnCode(api::ReturnCode::NOT_READY, "not ready")); std::unique_ptr<documentapi::CreateVisitorReply> reply( - dynamic_cast<documentapi::CreateVisitorReply*>( - cv.createReply().release())); + dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release())); CPPUNIT_ASSERT(reply.get()); - _converter->transferReplyState(cvr, *reply); - CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::ERROR_NODE_NOT_READY, reply->getError(0).getCode()); - - CPPUNIT_ASSERT_EQUAL(document::BucketId(INT_MAX), reply->getLastBucket()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(std::numeric_limits<int>::max()), reply->getLastBucket()); } void DocumentApiConverterTest::testCreateVisitorReplyLastBucket() { - documentapi::CreateVisitorMessage cv( - "mylib", - "myinstance", - "control-dest", - "data-dest"); - - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(cv, _repo); + documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest"); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); CPPUNIT_ASSERT(cmd.get()); api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd); - - api::CreateVisitorReply cvr(cvc); cvr.setLastBucket(document::BucketId(123)); - - std::unique_ptr<documentapi::CreateVisitorReply> reply( - dynamic_cast<documentapi::CreateVisitorReply*>( - cv.createReply().release())); + dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release())); CPPUNIT_ASSERT(reply.get()); - _converter->transferReplyState(cvr, *reply); - CPPUNIT_ASSERT_EQUAL(document::BucketId(123), reply->getLastBucket()); } @@ -282,8 +231,7 @@ void DocumentApiConverterTest::testDestroyVisitor() { documentapi::DestroyVisitorMessage cv("myinstance"); - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(cv, _repo); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo); api::DestroyVisitorCommand* pc = dynamic_cast<api::DestroyVisitorCommand*>(cmd.get()); @@ -302,8 +250,7 @@ DocumentApiConverterTest::testVisitorInfo() vicmd.setBucketsCompleted(bucketsCompleted); - std::unique_ptr<mbus::Message> mbusmsg = - _converter->toDocumentAPI(vicmd, _repo); + std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(vicmd, _repo); documentapi::VisitorInfoMessage* mbusvi = dynamic_cast<documentapi::VisitorInfoMessage*>(mbusmsg.get()); CPPUNIT_ASSERT(mbusvi); @@ -323,8 +270,7 @@ DocumentApiConverterTest::testVisitorInfo() void DocumentApiConverterTest::testDocBlock() { - Document::SP - doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); char buffer[10000]; vdslib::WritableDocumentList docBlock(_repo, buffer, sizeof(buffer)); @@ -335,11 +281,9 @@ DocumentApiConverterTest::testDocBlock() bucketId.setUsedBits(32); api::DocBlockCommand dbcmd(bucketId, docBlock, std::shared_ptr<void>()); - dbcmd.setTimeout(123456); - std::unique_ptr<mbus::Message> mbusmsg = - _converter->toDocumentAPI(dbcmd, _repo); + std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(dbcmd, _repo); documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get()); CPPUNIT_ASSERT(mbusdb); @@ -370,12 +314,10 @@ DocumentApiConverterTest::testDocBlockWithKeepTimeStamps() { CPPUNIT_ASSERT_EQUAL(dbcmd.keepTimeStamps(), false); - std::unique_ptr<mbus::Message> mbusmsg = - _converter->toDocumentAPI(dbcmd, _repo); + std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(dbcmd, _repo); documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get()); CPPUNIT_ASSERT(mbusdb); - CPPUNIT_ASSERT_EQUAL(mbusdb->keepTimeStamps(), false); } @@ -383,12 +325,10 @@ DocumentApiConverterTest::testDocBlockWithKeepTimeStamps() dbcmd.keepTimeStamps(true); CPPUNIT_ASSERT_EQUAL(dbcmd.keepTimeStamps(), true); - std::unique_ptr<mbus::Message> mbusmsg = - _converter->toDocumentAPI(dbcmd, _repo); + std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(dbcmd, _repo); documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get()); CPPUNIT_ASSERT(mbusdb); - CPPUNIT_ASSERT_EQUAL(mbusdb->keepTimeStamps(), true); } @@ -399,8 +339,7 @@ void DocumentApiConverterTest::testMultiOperation() { //create a document - Document::SP - doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); document::BucketIdFactory fac; document::BucketId bucketId = fac.getBucketId(doc->getId()); @@ -409,17 +348,13 @@ DocumentApiConverterTest::testMultiOperation() { documentapi::MultiOperationMessage momsg(_repo, bucketId, 10000); - vdslib::WritableDocumentList operations(_repo, &(momsg.getBuffer()[0]), - momsg.getBuffer().size()); + vdslib::WritableDocumentList operations(_repo, &(momsg.getBuffer()[0]), momsg.getBuffer().size()); operations.addPut(*doc, 100); - momsg.setOperations(operations); - CPPUNIT_ASSERT(momsg.getBuffer().size() > 0); // Convert it to Storage API - std::unique_ptr<api::StorageCommand> stcmd = - _converter->toStorageAPI(momsg, _repo); + std::unique_ptr<api::StorageCommand> stcmd = _converter->toStorageAPI(momsg, _repo); api::MultiOperationCommand* mocmd = dynamic_cast<api::MultiOperationCommand*>(stcmd.get()); CPPUNIT_ASSERT(mocmd); @@ -443,8 +378,7 @@ DocumentApiConverterTest::testMultiOperation() mocmd.getOperations().addPut(*doc, 100); // Convert it to documentapi - std::unique_ptr<mbus::Message> mbmsg = - _converter->toDocumentAPI(mocmd, _repo); + std::unique_ptr<mbus::Message> mbmsg = _converter->toDocumentAPI(mocmd, _repo); documentapi::MultiOperationMessage* momsg = dynamic_cast<documentapi::MultiOperationMessage*>(mbmsg.get()); CPPUNIT_ASSERT(momsg); @@ -473,33 +407,28 @@ DocumentApiConverterTest::testBatchDocumentUpdate() { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(_html_type, docId)); + document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId)); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(_html_type, docId)); + document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId)); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(_html_type, docId)); + document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId)); updates.push_back(update); } - std::shared_ptr<documentapi::BatchDocumentUpdateMessage> msg( - new documentapi::BatchDocumentUpdateMessage(1234)); + auto msg = std::make_shared<documentapi::BatchDocumentUpdateMessage>(1234); for (std::size_t i = 0; i < updates.size(); ++i) { msg->addUpdate(updates[i]); } - std::unique_ptr<storage::api::StorageCommand> cmd = - _converter->toStorageAPI(*msg, _repo); + std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(*msg, _repo); api::BatchDocumentUpdateCommand* batchCmd = dynamic_cast<api::BatchDocumentUpdateCommand*>(cmd.get()); CPPUNIT_ASSERT(batchCmd); CPPUNIT_ASSERT_EQUAL(updates.size(), batchCmd->getUpdates().size()); diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 974c756359f..3f1e9b69963 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -298,7 +298,7 @@ VisitorManagerTest::getMessagesAndReply( switch (session.sentMessages[i]->getType()) { case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: docs.push_back(static_cast<documentapi::PutDocumentMessage&>( - *session.sentMessages[i]).getDocument()); + *session.sentMessages[i]).getDocumentSP()); break; case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT: docIds.push_back(static_cast<documentapi::RemoveDocumentMessage&>( diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 85fbb6207bb..8abe7a3857d 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -333,26 +333,22 @@ VisitorTest::getMessagesAndReply( { vespalib::MonitorGuard guard(session.getMonitor()); CPPUNIT_ASSERT(!session.sentMessages.empty()); - std::unique_ptr<documentapi::DocumentMessage> msg( - std::move(session.sentMessages.front())); + std::unique_ptr<documentapi::DocumentMessage> msg(std::move(session.sentMessages.front())); session.sentMessages.pop_front(); CPPUNIT_ASSERT(msg->getPriority() < 16); switch (msg->getType()) { case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: docs.push_back( - static_cast<documentapi::PutDocumentMessage&>(*msg) - .getDocument()); + static_cast<documentapi::PutDocumentMessage&>(*msg).getDocumentSP()); break; case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT: docIds.push_back( - static_cast<documentapi::RemoveDocumentMessage&>(*msg) - .getDocumentId()); + static_cast<documentapi::RemoveDocumentMessage&>(*msg).getDocumentId()); break; case documentapi::DocumentProtocol::MESSAGE_VISITORINFO: infoMessages.push_back( - static_cast<documentapi::VisitorInfoMessage&>(*msg) - .getErrorMessage()); + static_cast<documentapi::VisitorInfoMessage&>(*msg).getErrorMessage()); break; default: break; diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 7dc5581d44e..9df177a32dd 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -36,7 +36,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_PUTDOCUMENT: { documentapi::PutDocumentMessage& from(static_cast<documentapi::PutDocumentMessage&>(fromMsg)); - api::PutCommand::UP to(new api::PutCommand(document::BucketId(0), from.getDocument(), from.getTimestamp())); + api::PutCommand::UP to(new api::PutCommand(document::BucketId(0), from.getDocumentSP(), from.getTimestamp())); to->setCondition(from.getCondition()); toMsg = std::move(to); break; @@ -44,7 +44,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: { documentapi::UpdateDocumentMessage& from(static_cast<documentapi::UpdateDocumentMessage&>(fromMsg)); - api::UpdateCommand::UP to(new api::UpdateCommand(document::BucketId(0), from.getDocumentUpdate(), + api::UpdateCommand::UP to(new api::UpdateCommand(document::BucketId(0), from.getDocumentUpdateSP(), from.getNewTimestamp())); to->setOldTimestamp(from.getOldTimestamp()); to->setCondition(from.getCondition()); diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index 2564c500cbc..f3595afe1e7 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -6,7 +6,6 @@ #include <vespa/document/base/testdocman.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/rpcmessagebus.h> -#include <fstream> #include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> #include <vespa/messagebus/staticthrottlepolicy.h> #include <vespa/messagebus/testlib/slobrok.h> @@ -14,17 +13,15 @@ #include <vespa/storageapi/mbusprot/storagereply.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/statusmetricconsumer.h> -#include <vespa/memfilepersistence/memfile/memfilecache.h> #include <tests/testhelper.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <tests/dummystoragelink.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/storageserver/app/distributorprocess.h> #include <vespa/storageserver/app/memfileservicelayerprocess.h> #include <vespa/vespalib/util/exceptions.h> #include <sys/time.h> +#include <fstream> #include <vespa/log/log.h> LOG_SETUP(".storageservertest"); @@ -388,9 +385,8 @@ namespace { if (msg->getType() == DocumentProtocol::MESSAGE_PUTDOCUMENT) { documentapi::PutDocumentMessage& putMsg( - static_cast<documentapi::PutDocumentMessage&>( - *msg)); - std::cerr << " - " << putMsg.getDocument()->getId(); + static_cast<documentapi::PutDocumentMessage&>(*msg)); + std::cerr << " - " << putMsg.getDocument().getId(); } std::cerr << "\n"; } @@ -419,8 +415,7 @@ namespace { } FastOS_Thread::Sleep(1); } - LOG(info, "Currently, we have received %u ok replies and have %u " - "pending ones.", + LOG(info, "Currently, we have received %u ok replies and have %u pending ones.", _processedOk, _currentPending); } }; diff --git a/travis/travis-build-cpp.sh b/travis/travis-build-cpp.sh index 6320cd505ca..42dbf0e6467 100755 --- a/travis/travis-build-cpp.sh +++ b/travis/travis-build-cpp.sh @@ -7,7 +7,7 @@ BUILD_DIR=~/build mkdir "${BUILD_DIR}" -export CCACHE_SIZE="4G" +export CCACHE_SIZE="1G" export CCACHE_COMPRESS=1 NUM_THREADS=4 |