aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBjørn Christian Seime <bjorn.christian@seime.no>2017-08-25 13:31:06 +0200
committerGitHub <noreply@github.com>2017-08-25 13:31:06 +0200
commitd5d2098a6fa163e9a88f2ab09471ce6380b189f9 (patch)
treedd677f1aa2f89cfc5aad5dd223fd273c3a9c6274
parentc34d8ea5a1528c7f8098c1b61d0c7b8a4354fe1d (diff)
parent56aa0fadd0464e66ad80247e1d92bce5500b584d (diff)
Merge branch 'master' into geirst/sort-document-types-in-topological-order
-rw-r--r--.travis.yml1
-rw-r--r--CONTRIBUTING.md2
-rw-r--r--container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java7
-rw-r--r--controller-api/OWNERS2
-rw-r--r--controller-api/pom.xml181
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java42
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java49
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java98
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java32
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java47
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java27
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java16
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java17
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java42
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java43
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java58
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java55
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java34
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java32
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java18
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java16
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java14
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java20
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java35
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java47
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java48
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java31
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java36
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java22
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java21
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java58
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java39
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java17
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java32
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java48
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java44
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java38
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java18
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java23
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java38
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java39
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java73
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java41
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java29
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java29
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java69
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java103
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java19
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java21
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java15
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java29
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java15
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java18
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java30
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java22
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java34
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java17
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java33
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java136
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java182
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java66
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java16
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java17
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java24
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java59
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java48
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java51
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java21
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java12
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java23
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java35
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java23
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java19
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java73
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java95
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java68
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java55
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java131
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java8
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java35
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java42
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java112
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java110
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java118
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java74
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java25
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java32
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java20
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java40
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java20
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java69
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java41
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java15
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java22
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java105
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java21
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java182
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java53
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java93
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java31
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java24
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java74
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java27
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java28
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java37
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java48
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java56
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java18
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java20
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java86
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java45
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java22
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java50
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java16
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java20
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java30
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java19
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java31
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java87
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java26
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java31
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java14
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java39
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java24
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java35
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java64
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java110
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java27
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java31
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java23
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java5
-rw-r--r--controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java31
-rw-r--r--controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java153
-rw-r--r--controller-server/OWNERS2
-rw-r--r--controller-server/pom.xml211
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java26
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java276
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java541
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java273
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java32
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java238
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java37
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java57
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java23
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java147
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java8
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java200
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java75
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java60
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java90
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java50
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java333
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java209
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java48
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java63
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java24
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java55
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java34
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java368
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java100
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java67
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java24
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java66
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java234
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java42
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java67
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java80
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java118
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java32
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java94
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java33
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java5
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java304
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java74
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java201
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java45
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java132
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java18
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java19
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java44
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java66
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java31
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java109
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java42
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java96
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java38
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java26
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java64
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java1065
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java43
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java164
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java117
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java25
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java31
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java72
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java191
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java84
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java46
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java122
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java26
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java68
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java55
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java16
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java33
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java27
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java44
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java23
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java50
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java31
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java168
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java62
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java182
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java139
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java55
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java8
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java148
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java54
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java48
-rw-r--r--controller-server/src/main/resources/WEB-INF/web.xml8
-rw-r--r--controller-server/src/main/resources/configdefinitions/http-access-control.def4
-rw-r--r--controller-server/src/main/resources/configdefinitions/maintainer.def4
-rw-r--r--controller-server/src/main/resources/configdefinitions/rotations.def4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java204
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java601
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java202
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java83
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java38
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java74
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java44
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java41
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java113
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java209
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java167
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java179
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java106
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java64
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java22
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java46
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java280
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java169
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java93
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java132
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java56
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java304
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java33
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java196
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java101
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java136
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java85
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java23
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java598
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java78
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java91
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java63
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java95
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json104
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json0
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json9
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json57
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json1
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json9
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json9
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json22
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json18
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json18
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json7
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json18
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json11
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json13
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json12
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json9
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json8
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java40
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json31
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json7
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java72
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json50
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java79
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java112
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json41
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java165
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java26
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java283
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java65
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java200
-rw-r--r--controller-server/src/test/resources/chef_output.json34
-rw-r--r--controller-server/src/test/resources/job-grandparent.json4
-rw-r--r--controller-server/src/test/resources/job-parent.json9
-rw-r--r--controller-server/src/test/resources/job.json9
-rw-r--r--documentapi/src/tests/messagebus/messagebus_test.cpp2
-rw-r--r--documentapi/src/tests/messages/messages50test.cpp10
-rw-r--r--documentapi/src/tests/messages/messages52test.cpp9
-rw-r--r--documentapi/src/tests/policies/policies_test.cpp2
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp25
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h24
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp19
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h33
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp19
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h25
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp6
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp10
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp4
-rw-r--r--documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp19
-rw-r--r--jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java31
-rw-r--r--messagebus/src/vespa/messagebus/network/rpcnetwork.cpp1
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java2
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java6
-rw-r--r--pom.xml2
-rw-r--r--searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp12
-rw-r--r--searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp2
-rw-r--r--searchcore/src/tests/proton/documentdb/documentdb_test.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/grouping/groupingcontext.h8
-rw-r--r--searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp17
-rw-r--r--searchcore/src/vespa/searchcore/grouping/groupingmanager.h22
-rw-r--r--searchcore/src/vespa/searchcore/grouping/groupingsession.cpp26
-rw-r--r--searchcore/src/vespa/searchcore/grouping/groupingsession.h43
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp7
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/match_thread.h6
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/matcher.cpp1
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/result_processor.h7
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp7
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h11
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.h5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h9
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/matchview.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/matchview.h13
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchview.h3
-rw-r--r--searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp53
-rw-r--r--searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/aggregation/group.h2
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp6
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp14
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h5
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp136
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_search_context.h5
-rw-r--r--searchlib/src/vespa/searchlib/attribute/reference_attribute.h4
-rw-r--r--searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp6
-rw-r--r--searchlib/src/vespa/searchlib/attribute/reference_mappings.h10
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp6
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/emptysearch.h1
-rw-r--r--storage/src/tests/storageserver/documentapiconvertertest.cpp145
-rw-r--r--storage/src/tests/visiting/visitormanagertest.cpp2
-rw-r--r--storage/src/tests/visiting/visitortest.cpp12
-rw-r--r--storage/src/vespa/storage/storageserver/documentapiconverter.cpp4
-rw-r--r--storageserver/src/tests/storageservertest.cpp13
-rwxr-xr-xtravis/travis-build-cpp.sh2
413 files changed, 22031 insertions, 422 deletions
diff --git a/.travis.yml b/.travis.yml
index 6e03d4f73b2..29aa44a4e9c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,6 +21,7 @@ branches:
before_cache:
- sudo rm -rf $HOME/.m2/repository/com/yahoo
- du --summarize --human-readable $HOME/.m2/repository
+ - du --summarize --human-readable $HOME/.ccache
- ccache --show-stats
install: true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 33fe2ea4f1a..ddc316e5b9c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -34,7 +34,7 @@ refer to the [APIs](http://docs.vespa.ai/documentation/api.html).
## Where to start contributing
Most features plug into the [Vespa Container](docs.vespa.ai/documentation/jdisc/index.html) -
-this is the most likely pleace to write enhancements.
+this is the most likely place to write enhancements.
Discuss with the community if others have similar feature requests - make the feature generic.
### Getting started
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
index fa2ee8e89a9..e59f012856a 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
@@ -42,6 +42,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -79,7 +80,7 @@ public final class ConfiguredApplication implements Application {
new ComponentRegistry<>(),
new ComponentRegistry<>());
private final OsgiFramework restrictedOsgiFramework;
- private volatile int applicationSerialNo = 0;
+ private final AtomicInteger applicationSerialNo = new AtomicInteger(0);
private HandlersConfigurerDi configurer;
private ScheduledThreadPoolExecutor shutdownDeadlineExecutor;
private Thread reconfigurerThread;
@@ -172,10 +173,12 @@ public final class ConfiguredApplication implements Application {
startAndStopServers();
log.info("Switching to the latest deployed set of configurations and components. " +
- "Application switch number: " + (applicationSerialNo++));
+ "Application switch number: " + applicationSerialNo.getAndIncrement());
}
private ContainerBuilder createBuilderWithGuiceBindings() {
+ log.info("Initializing new set of configurations and components. " +
+ "Application switch number: " + applicationSerialNo.get());
ContainerBuilder builder = activator.newContainerBuilder();
setupGuiceBindings(builder.guiceModules());
return builder;
diff --git a/controller-api/OWNERS b/controller-api/OWNERS
new file mode 100644
index 00000000000..e6a0537ba53
--- /dev/null
+++ b/controller-api/OWNERS
@@ -0,0 +1,2 @@
+bratseth
+mpolden
diff --git a/controller-api/pom.xml b/controller-api/pom.xml
new file mode 100644
index 00000000000..51666da0c03
--- /dev/null
+++ b/controller-api/pom.xml
@@ -0,0 +1,181 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ </parent>
+ <artifactId>controller-api</artifactId>
+ <packaging>container-plugin</packaging>
+ <version>6-SNAPSHOT</version>
+
+ <dependencies>
+
+ <!-- provided -->
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>component</artifactId>
+ <scope>provided</scope>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>annotations</artifactId>
+ <scope>provided</scope>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespajlib</artifactId>
+ <scope>provided</scope>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>serviceview</artifactId>
+ <scope>provided</scope>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-provisioning</artifactId>
+ <scope>provided</scope>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jdk8</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-multipart</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>${jersey2.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <classifier>no_aop</classifier>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- compile -->
+
+ <dependency>
+ <groupId>com.intellij</groupId>
+ <artifactId>annotations</artifactId>
+ <version>9.0.4</version>
+ </dependency>
+
+ <!-- test -->
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>configdefinitions</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:all</arg>
+ <arg>-Xlint:-serial</arg>
+ <arg>-Werror</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <useCommonAssemblyIds>false</useCommonAssemblyIds>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-deploy.jar</file>
+ <type>jar</type>
+ <classifier>deploy</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java
new file mode 100644
index 00000000000..4233f6308d5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationApi.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.AthensDomainsResponse;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantInfo;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantPipelinesInfo;
+
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+
+/**
+ * @author gv
+ */
+@Path("/v4/")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+public interface ApplicationApi {
+
+ @GET
+ @Path(TenantResource.API_PATH)
+ List<TenantInfo> listTenants();
+
+ @Path(TenantResource.API_PATH + "/{tenantId}")
+ TenantResource tenant(@PathParam("tenantId")TenantId tenantId);
+
+ @GET
+ @Path("athensDomain")
+ AthensDomainsResponse listAthensDomains(@DefaultValue("") @QueryParam("prefix") String prefix);
+
+ @GET
+ @Path("tenant-pipeline")
+ TenantPipelinesInfo listTenantPipelines();
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java
new file mode 100644
index 00000000000..e5833682c90
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ApplicationResource.java
@@ -0,0 +1,49 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.JobStatusList;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ApplicationReference;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.InstancesReply;
+
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+
+/**
+ * @author gv
+ */
+@Path("") //Ensures that the produces annotation is inherited
+@Produces(MediaType.APPLICATION_JSON)
+public interface ApplicationResource {
+
+ String API_PATH = "application";
+
+ @GET
+ List<ApplicationReference> listApplications();
+
+ @Path("{applicationId}")
+ @POST
+ ApplicationReference createApplication(@PathParam("applicationId") ApplicationId applicationId);
+
+ @Path("{applicationId}")
+ @DELETE
+ void deleteApplication(@PathParam("applicationId") ApplicationId applicationId);
+
+ @Path("{applicationId}/environment")
+ EnvironmentResource environment();
+
+ @Path("{applicationId}")
+ @GET
+ InstancesReply listInstances(@PathParam("applicationId") ApplicationId applicationId);
+
+ @Path("{applicationId}/deployment")
+ @GET
+ JobStatusList deployment(@PathParam("applicationId") ApplicationId applicationId);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java
new file mode 100644
index 00000000000..4f1583dd905
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java
@@ -0,0 +1,98 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployResult;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.InstanceInformation;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import org.glassfish.jersey.media.multipart.FormDataParam;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import java.io.InputStream;
+
+/**
+ * @author Tony Vaagenes
+ * @author gv
+ */
+@Path("") //Ensures that the produces annotation is inherited
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public interface EnvironmentResource {
+
+ String API_PATH = "environment";
+
+ String APPLICATION_ZIP = "applicationZip";
+ String DEPLOY_OPTIONS = "deployOptions";
+
+ @POST
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}/deploy")
+ @Consumes({MediaType.MULTIPART_FORM_DATA})
+ DeployResult deploy(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId,
+ @FormDataParam(APPLICATION_ZIP) InputStream applicationZipFile,
+ @FormDataParam(APPLICATION_ZIP) FormDataContentDisposition fileMetaData,
+ @FormDataParam(DEPLOY_OPTIONS) FormDataBodyPart deployOptions);
+
+ @DELETE
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}")
+ String deactivate(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId);
+
+ @POST
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}/restart")
+ String restart(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId,
+ @QueryParam("hostname") Hostname hostname);
+
+ @GET
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}")
+ InstanceInformation instanceInfo(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId);
+
+ @GET
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}/converge")
+ JsonNode waitForConfigConverge(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId,
+ @QueryParam("timeout") long timeoutInSeconds);
+
+ @POST
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}/log")
+ JsonNode grabLog(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId);
+
+ @Path("{environmentId}/region/{regionId}/instance/{instanceId}/service")
+ ServiceViewResource service();
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java
new file mode 100644
index 00000000000..c058a72341a
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/ServiceViewResource.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.util.HashMap;
+
+/**
+ * @author Stian Kristoffersen
+ */
+@Path("")
+@Produces(MediaType.APPLICATION_JSON)
+public interface ServiceViewResource {
+
+ @GET
+ @Path("")
+ @Produces(MediaType.APPLICATION_JSON)
+ ApplicationView getUserInfo();
+
+ @GET
+ @Path("{serviceIdentifier}/{apiParams: .*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("rawtypes")
+ HashMap singleService(@PathParam("serviceIdentifier") String identifier,
+ @PathParam("apiParams") String apiParams);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java
new file mode 100644
index 00000000000..8db6f982ef6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/TenantResource.java
@@ -0,0 +1,47 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantCreateOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantInfo;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantMigrateOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantUpdateOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantWithApplications;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * @author Tony Vaagenes
+ */
+@Path("") //Ensures that the produces annotation is inherited
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public interface TenantResource {
+
+ String API_PATH = "tenant";
+
+ @GET
+ TenantWithApplications metaData();
+
+ @DELETE
+ TenantInfo deleteTenant();
+
+ @POST
+ TenantInfo createTenant(TenantCreateOptions tenantOptions);
+
+ @PUT
+ TenantInfo updateTenant(TenantUpdateOptions tenantOptions);
+
+ @Path(ApplicationResource.API_PATH)
+ ApplicationResource application();
+
+ @PUT
+ @Path("migrateTenantToAthens")
+ TenantInfo migrateTenantToAthens(TenantMigrateOptions tenantOptions);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java
new file mode 100644
index 00000000000..a290323a245
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java
@@ -0,0 +1,27 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.UserInfo;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * @author gv
+ */
+@Path("/v4/user")
+@Produces(MediaType.APPLICATION_JSON)
+public interface UserResource {
+ @GET
+ @JsonInclude(value = JsonInclude.Include.NON_NULL)
+ UserInfo whoAmI(@QueryParam("userOverride") UserId userOverride);
+
+ @PUT
+ void createUserTenant();
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java
new file mode 100644
index 00000000000..c542987e78f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ApplicationReference.java
@@ -0,0 +1,16 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+
+import java.net.URI;
+
+/**
+ * @author Stian Kristoffersen
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ApplicationReference {
+ public ApplicationId application;
+ public URI url;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java
new file mode 100644
index 00000000000..400b973a4e1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/AthensDomainsResponse.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+
+import java.util.List;
+
+/**
+ * @author gv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AthensDomainsResponse extends JsonResponse<List<AthensDomain>> {
+ public AthensDomainsResponse(List<AthensDomain> athensDomainList) {
+ super(athensDomainList);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java
new file mode 100644
index 00000000000..d8551898f7c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployOptions.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.component.Version;
+
+import java.util.Optional;
+
+/**
+ * @author gjoranv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DeployOptions {
+
+ public final Optional<ScrewdriverBuildJob> screwdriverBuildJob;
+ public final Optional<String> vespaVersion;
+ public final boolean ignoreValidationErrors;
+ public final boolean deployCurrentVersion;
+
+ @JsonCreator
+ public DeployOptions(@JsonProperty("screwdriverBuildJob") Optional<ScrewdriverBuildJob> screwdriverBuildJob,
+ @JsonProperty("vespaVersion") Optional<Version> vespaVersion,
+ @JsonProperty("ignoreValidationErrors") boolean ignoreValidationErrors,
+ @JsonProperty("deployCurrentVersion") boolean deployCurrentVersion) {
+ this.screwdriverBuildJob = screwdriverBuildJob;
+ this.vespaVersion = vespaVersion.map(Version::toString);
+ this.ignoreValidationErrors = ignoreValidationErrors;
+ this.deployCurrentVersion = deployCurrentVersion;
+ }
+
+ @Override
+ public String toString() {
+ return "DeployData{" +
+ "screwdriverBuildJob=" + screwdriverBuildJob.map(ScrewdriverBuildJob::toString).orElse("None") +
+ ", vespaVersion=" + vespaVersion.orElse("None") +
+ ", ignoreValidationErrors=" + ignoreValidationErrors +
+ ", deployCurrentVersion=" + deployCurrentVersion +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java
new file mode 100644
index 00000000000..3a98926805f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeployResult.java
@@ -0,0 +1,43 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
+
+import java.util.List;
+
+/**
+ * @author gjoranv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DeployResult {
+
+ public final RevisionId revisionId;
+ public final Long applicationZipSize;
+ public final List<LogEntry> prepareMessages;
+ public final ConfigChangeActions configChangeActions;
+
+ @JsonCreator
+ public DeployResult(@JsonProperty("revisionId") RevisionId revisionId,
+ @JsonProperty("applicationZipSize") Long applicationZipSize,
+ @JsonProperty("prepareMessages") List<LogEntry> prepareMessages,
+ @JsonProperty("configChangeActions") ConfigChangeActions configChangeActions) {
+ this.revisionId = revisionId;
+ this.applicationZipSize = applicationZipSize;
+ this.prepareMessages = prepareMessages;
+ this.configChangeActions = configChangeActions;
+ }
+
+ @Override
+ public String toString() {
+ return "DeployResult{" +
+ "revisionId=" + revisionId.id() +
+ ", applicationZipSize=" + applicationZipSize +
+ ", prepareMessages=" + prepareMessages +
+ ", configChangeActions=" + configChangeActions +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java
new file mode 100644
index 00000000000..d014a82bf62
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java
@@ -0,0 +1,58 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+/**
+ * Represent the operational status of a service endpoint (where the endpoint itself
+ * is identified by the container cluster id).
+ *
+ * The status of an endpoint may be assigned from the controller.
+ *
+ * @author smorgrav
+ */
+public class EndpointStatus {
+ private final String agent;
+ private final String reason;
+ private final Status status;
+ private final long epoch;
+
+ public enum Status {
+ in,
+ out,
+ unknown;
+ }
+
+ public EndpointStatus(Status status, String reason, String agent, long epoch) {
+ this.status = status;
+ this.reason = reason;
+ this.agent = agent;
+ this.epoch = epoch;
+ }
+
+ /**
+ * @return The agent responsible setting this status
+ */
+ public String getAgent() {
+ return agent;
+ }
+
+ /**
+ * @return The reason for this status (e.g. 'incident INCXXX')
+ */
+ public String getReason() {
+ return reason;
+ }
+
+ /**
+ * @return The current status
+ */
+ public Status getStatus() {
+ return status;
+ }
+
+ /**
+ * @return The epoch for when this status became active
+ */
+ public long getEpoch() {
+ return epoch;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java
new file mode 100644
index 00000000000..317da739103
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/GitRevision.java
@@ -0,0 +1,55 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+
+import java.util.Objects;
+
+/**
+ * @author gv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class GitRevision {
+
+ public final GitRepository repository;
+ public final GitBranch branch;
+ public final GitCommit commit;
+
+ @JsonCreator
+ public GitRevision(@JsonProperty("repository") GitRepository repository,
+ @JsonProperty("branch") GitBranch branch,
+ @JsonProperty("commit") GitCommit commit) {
+ this.repository = repository;
+ this.branch = branch;
+ this.commit = commit;
+ }
+
+ @Override
+ public String toString() {
+ return "GitRevision{" +
+ "repository=" + repository +
+ ", branch=" + branch +
+ ", commit=" + commit +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ GitRevision that = (GitRevision) o;
+ return Objects.equals(repository, that.repository) &&
+ Objects.equals(branch, that.branch) &&
+ Objects.equals(commit, that.commit);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(repository, branch, commit);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java
new file mode 100644
index 00000000000..e862bd744dc
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java
@@ -0,0 +1,34 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yahoo.vespa.hosted.controller.api.cost.CostJsonModel;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+
+import java.net.URI;
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InstanceInformation {
+ public List<URI> serviceUrls;
+ public URI nodes;
+ public URI elkUrl;
+ public URI yamasUrl;
+ public RevisionId revision;
+ public Long deployTimeEpochMs;
+ public Long expiryTimeEpochMs;
+
+ public ScrewdriverId screwdriverId;
+ public GitRepository gitRepository;
+ public GitBranch gitBranch;
+ public GitCommit gitCommit;
+
+ public CostJsonModel.Application cost;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java
new file mode 100644
index 00000000000..6ac27d0bad9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceReference.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yahoo.vespa.hosted.controller.api.bcp.BcpStatus;
+import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId;
+
+import java.net.URI;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InstanceReference {
+ public EnvironmentId environment;
+ public RegionId region;
+ public InstanceId instance;
+ public BcpStatus bcpStatus;
+
+ public URI url;
+
+ public static InstanceReference createInstanceReference(InstanceId instanceId, RegionId regionId, EnvironmentId environmentId, URI uri) {
+ InstanceReference instanceReference = new InstanceReference();
+ instanceReference.instance = instanceId;
+ instanceReference.region = regionId;
+ instanceReference.environment = environmentId;
+ instanceReference.url = uri;
+ return instanceReference;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
new file mode 100644
index 00000000000..ff0c155460e
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author Tony Vaagenes
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InstancesReply {
+ public Set<URI> globalRotations;
+ public List<InstanceReference> instances;
+ public String compileVersion;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java
new file mode 100644
index 00000000000..cce8a8c88fc
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatus.java
@@ -0,0 +1,16 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * @author bratseth
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class JobStatus {
+
+ public String jobType;
+ public long lastCompleted;
+ public boolean success;
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java
new file mode 100644
index 00000000000..30af3291fbd
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JobStatusList.java
@@ -0,0 +1,14 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.List;
+
+/**
+ * @author bratseth
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class JobStatusList {
+ public List<JobStatus> jobs;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java
new file mode 100644
index 00000000000..3690644c49b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/JsonResponse.java
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+
+/**
+ * @author gv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(Include.NON_NULL)
+public abstract class JsonResponse<DATA> {
+ public DATA data;
+
+ public JsonResponse(DATA data) {
+ this.data = data;
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java
new file mode 100644
index 00000000000..d5fc0addd70
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/LogEntry.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author gjoranv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class LogEntry {
+
+ public final long time;
+ public final String level;
+ public final String message;
+
+ @JsonCreator
+ public LogEntry(@JsonProperty("time") long time,
+ @JsonProperty("level") String level,
+ @JsonProperty("message") String message) {
+ this.time = time;
+ this.level = level;
+ this.message = message;
+ }
+
+ @Override
+ public String toString() {
+ return "LogEntry{" +
+ "time=" + time +
+ ", level='" + level + '\'' +
+ ", message='" + message + '\'' +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java
new file mode 100644
index 00000000000..032b97c5424
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ScrewdriverBuildJob.java
@@ -0,0 +1,47 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+
+import java.util.Objects;
+
+/**
+ * @author gjoranv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ScrewdriverBuildJob {
+ public final ScrewdriverId screwdriverId;
+ public final GitRevision gitRevision;
+
+ @JsonCreator
+ public ScrewdriverBuildJob(@JsonProperty("screwdriverId") ScrewdriverId screwdriverId,
+ @JsonProperty("gitRevision") GitRevision gitRevision) {
+ this.screwdriverId = screwdriverId;
+ this.gitRevision = gitRevision;
+ }
+
+ @Override
+ public String toString() {
+ return "ScrewdriverBuildJob{" +
+ "screwdriverId=" + screwdriverId +
+ ", gitRevision=" + gitRevision +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ScrewdriverBuildJob that = (ScrewdriverBuildJob) o;
+ return Objects.equals(screwdriverId, that.screwdriverId) &&
+ Objects.equals(gitRevision, that.gitRevision);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(screwdriverId, gitRevision);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java
new file mode 100644
index 00000000000..4032a960b3c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantCreateOptions.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(value = JsonInclude.Include.NON_NULL)
+public class TenantCreateOptions {
+ public AthensDomain athensDomain;
+ public Property property;
+ public PropertyId propertyId;
+ public UserGroup userGroup;
+
+ public TenantCreateOptions() {}
+
+ public TenantCreateOptions(UserGroup userGroup, Property property, PropertyId propertyId) {
+ this.userGroup = userGroup;
+ this.property = property;
+ this.propertyId = propertyId;
+ }
+
+ public TenantCreateOptions(AthensDomain athensDomain, Property property, PropertyId propertyId) {
+ this.athensDomain = athensDomain;
+ this.property = property;
+ this.propertyId = propertyId;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("options: ");
+ sb.append("athens-domain='").append(this.athensDomain).append("', ");
+ sb.append("property='").append(this.property).append("'");
+ if (this.propertyId != null) {
+ sb.append(", propertyId='").append(this.propertyId).append("'");
+ }
+
+ return sb.toString();
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java
new file mode 100644
index 00000000000..ef1afbc9edf
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantInfo.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+
+import java.net.URI;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(value = JsonInclude.Include.NON_EMPTY)
+public class TenantInfo {
+ public TenantId tenant;
+ // TODO: make optional
+ public TenantMetaData metaData;
+ public URI url;
+
+ // Required for Jackson deserialization
+ public TenantInfo() {}
+
+ public TenantInfo(TenantId tenantId, TenantMetaData metaData, URI url) {
+ this.tenant = tenantId;
+ this.metaData = metaData;
+ this.url = url;
+ }
+
+ public TenantInfo(TenantId tenant, URI url) {
+ this.tenant = tenant;
+ this.url = url;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java
new file mode 100644
index 00000000000..5ded1d8030e
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMetaData.java
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+
+import java.util.Optional;
+
+/**
+ * @author gv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(value = Include.NON_EMPTY)
+public class TenantMetaData {
+ public TenantType type;
+ public Optional<AthensDomain> athensDomain;
+ public Optional<Property> property;
+ public Optional<UserGroup> userGroup;
+
+ // Required for Jackson deserialization
+ public TenantMetaData() {}
+
+ public TenantMetaData(TenantType type,
+ Optional<AthensDomain> athensDomain,
+ Optional<Property> property,
+ Optional<UserGroup> userGroup) {
+ this.type = type;
+ this.athensDomain = athensDomain;
+ this.property = property;
+ this.userGroup = userGroup;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java
new file mode 100644
index 00000000000..9c748eafd38
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantMigrateOptions.java
@@ -0,0 +1,22 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(value = JsonInclude.Include.NON_NULL)
+public class TenantMigrateOptions {
+
+ public AthensDomain athensDomain;
+
+ public TenantMigrateOptions() {}
+
+ public TenantMigrateOptions(AthensDomain athensDomain) {
+ this.athensDomain = athensDomain;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java
new file mode 100644
index 00000000000..a7f1fb408fe
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantPipelinesInfo.java
@@ -0,0 +1,21 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TenantPipelinesInfo {
+ public List<TenantPipelineInfo> tenantPipelines = new ArrayList<>();
+
+ public static class TenantPipelineInfo {
+ public String screwdriverId;
+ public String tenant;
+ public String application;
+ public String instance;
+ }
+
+ public List<TenantPipelineInfo> brokenTenantPipelines = new ArrayList<>();
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java
new file mode 100644
index 00000000000..2c543af7bf8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+/**
+ * @author bjorncs
+ */
+public enum TenantType {
+ OPSDB,
+ USER,
+ ATHENS
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java
new file mode 100644
index 00000000000..9b2f24e2f62
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantUpdateOptions.java
@@ -0,0 +1,58 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * @author gv
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(value = JsonInclude.Include.NON_ABSENT)
+public class TenantUpdateOptions {
+ public final Property property;
+ public final Optional<UserGroup> userGroup;
+ public final Optional<AthensDomain> athensDomain;
+
+ @JsonCreator
+ public TenantUpdateOptions(@JsonProperty("property") Property property,
+ @JsonProperty("userGroup") Optional<UserGroup> userGroup,
+ @JsonProperty("athensDomain") Optional<AthensDomain> athensDomain) {
+ this.userGroup = userGroup;
+ this.property = property;
+ this.athensDomain = athensDomain;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ TenantUpdateOptions that = (TenantUpdateOptions) o;
+ return Objects.equals(property, that.property) &&
+ Objects.equals(userGroup, that.userGroup) &&
+ Objects.equals(athensDomain, that.athensDomain);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(property, userGroup, athensDomain);
+ }
+
+ @Override
+ public String toString() {
+ return "TenantUpdateOptions{" +
+ "property=" + property +
+ ", userGroup=" + userGroup +
+ ", athensDomain=" + athensDomain +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java
new file mode 100644
index 00000000000..de731d5c971
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantWithApplications.java
@@ -0,0 +1,39 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+
+import java.util.List;
+
+/**
+ * @author Tony Vaagenes
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(value = JsonInclude.Include.NON_NULL)
+public class TenantWithApplications {
+ // TODO: use TenantMetaData instead of individual fields (requires dashboard updates)
+ public TenantType type;
+ public AthensDomain athensDomain;
+ public Property property;
+ public UserGroup userGroup;
+ public List<ApplicationReference> applications;
+
+ public TenantWithApplications() {}
+
+ public TenantWithApplications(
+ TenantType type,
+ AthensDomain athensDomain,
+ Property property,
+ UserGroup userGroup,
+ List<ApplicationReference> applications) {
+ this.type = type;
+ this.athensDomain = athensDomain;
+ this.property = property;
+ this.userGroup = userGroup;
+ this.applications = applications;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java
new file mode 100644
index 00000000000..2b2a089c543
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+import java.util.List;
+
+/**
+ * @author gv
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UserInfo {
+ public UserId user;
+ public boolean tenantExists;
+ public List<TenantInfo> tenants;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java
new file mode 100644
index 00000000000..397461a829d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ConfigChangeActions.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ConfigChangeActions {
+ @JsonProperty("restart") public final List<RestartAction> restartActions;
+ @JsonProperty("refeed") public final List<RefeedAction> refeedActions;
+
+ @JsonCreator
+ public ConfigChangeActions(@JsonProperty("restart") List<RestartAction> restartActions,
+ @JsonProperty("refeed") List<RefeedAction> refeedActions) {
+ this.restartActions = restartActions;
+ this.refeedActions = refeedActions;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigChangeActions{" +
+ "restartActions=" + restartActions +
+ ", refeedActions=" + refeedActions +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java
new file mode 100644
index 00000000000..0546a3b5c44
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RefeedAction.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RefeedAction {
+ public final String name;
+ public final boolean allowed;
+ public final String documentType;
+ public final String clusterName;
+ public final List<ServiceInfo> services;
+ public final List<String> messages;
+
+ @JsonCreator
+ public RefeedAction(@JsonProperty("name") String name,
+ @JsonProperty("allowed") boolean allowed,
+ @JsonProperty("documentType") String documentType,
+ @JsonProperty("clusterName") String clusterName,
+ @JsonProperty("services") List<ServiceInfo> services,
+ @JsonProperty("messages") List<String> messages) {
+ this.name = name;
+ this.allowed = allowed;
+ this.documentType = documentType;
+ this.clusterName = clusterName;
+ this.services = services;
+ this.messages = messages;
+ }
+
+ @Override
+ public String toString() {
+ return "RefeedAction{" +
+ "name='" + name + '\'' +
+ ", allowed=" + allowed +
+ ", documentType='" + documentType + '\'' +
+ ", clusterName='" + clusterName + '\'' +
+ ", services=" + services +
+ ", messages=" + messages +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java
new file mode 100644
index 00000000000..a760a26d47d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/RestartAction.java
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RestartAction {
+ public final String clusterName;
+ public final String clusterType;
+ public final String serviceType;
+ public final List<ServiceInfo> services;
+ public final List<String> messages;
+
+ @JsonCreator
+ public RestartAction(@JsonProperty("clusterName") String clusterName,
+ @JsonProperty("clusterType") String clusterType,
+ @JsonProperty("serviceType") String serviceType,
+ @JsonProperty("services") List<ServiceInfo> services,
+ @JsonProperty("messages") List<String> messages) {
+ this.clusterName = clusterName;
+ this.clusterType = clusterType;
+ this.serviceType = serviceType;
+ this.services = services;
+ this.messages = messages;
+ }
+
+ @Override
+ public String toString() {
+ return "RestartAction{" +
+ "clusterName='" + clusterName + '\'' +
+ ", clusterType='" + clusterType + '\'' +
+ ", serviceType='" + serviceType + '\'' +
+ ", services=" + services +
+ ", messages=" + messages +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java
new file mode 100644
index 00000000000..8d03d2da440
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/ServiceInfo.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author bjorncs
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ServiceInfo {
+ public final String serviceName;
+ public final String serviceType;
+ public final String configId;
+ public final String hostName;
+
+ @JsonCreator
+ public ServiceInfo(@JsonProperty("serviceName") String serviceName,
+ @JsonProperty("serviceType") String serviceType,
+ @JsonProperty("configId") String configId,
+ @JsonProperty("hostName")String hostName) {
+ this.serviceName = serviceName;
+ this.serviceType = serviceType;
+ this.configId = configId;
+ this.hostName = hostName;
+ }
+
+ @Override
+ public String toString() {
+ return "ServiceInfo{" +
+ "serviceName='" + serviceName + '\'' +
+ ", serviceType='" + serviceType + '\'' +
+ ", configId='" + configId + '\'' +
+ ", hostName='" + hostName + '\'' +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java
new file mode 100644
index 00000000000..1201f148329
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/configserverbindings/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java
new file mode 100644
index 00000000000..1eac6d8c296
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java
new file mode 100644
index 00000000000..e7b71b693a3
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.application.v4;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java
new file mode 100644
index 00000000000..679d5fc5727
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.bcp;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class BcpStatus {
+ public String rotationStatus;
+ public String reason;
+
+ // For jackson
+ public BcpStatus() {}
+
+ public BcpStatus(String rotationStatus, String reason) {
+ this.rotationStatus = rotationStatus;
+ this.reason = reason;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java
new file mode 100644
index 00000000000..c77f9fceef9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BrooklynStatusResource.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.bcp;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * @author andreer
+ */
+@Path("") //Ensures that the produces annotation is inherited
+@Produces(MediaType.APPLICATION_JSON)
+public interface BrooklynStatusResource {
+
+ @GET
+ @Path("{rotation}")
+ @Produces(MediaType.APPLICATION_JSON)
+ JsonNode rotationStatus(@PathParam("rotation") String page);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java
new file mode 100644
index 00000000000..2bb442c3db8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.bcp;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java
new file mode 100644
index 00000000000..ad28d3ca5b5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Environment.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.configserver;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+/**
+ * Environment representation using the same definition as configserver. And allowing
+ * serialization/deserialization to/from JSON.
+ *
+ * @author Ulf Lilleengen
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Environment {
+ private final com.yahoo.config.provision.Environment environment;
+
+ public Environment(com.yahoo.config.provision.Environment environment) {
+ this.environment = environment;
+ }
+
+ @JsonValue
+ public String value() {
+ return environment.value();
+ }
+
+ @Override
+ public String toString() {
+ return value();
+ }
+
+ public com.yahoo.config.provision.Environment getEnvironment() {
+ return environment;
+ }
+
+ public Environment(String environment) {
+ this.environment = com.yahoo.config.provision.Environment.from(environment);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java
new file mode 100644
index 00000000000..b7f1560eb67
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/Region.java
@@ -0,0 +1,39 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.configserver;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.yahoo.config.provision.RegionName;
+
+/**
+ * Region representation using the same definition as configserver. And allowing
+ * serialization/deserialization to/from JSON.
+ *
+ * @author Ulf Lilleengen
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Region {
+ private final RegionName region;
+
+ public Region(RegionName region) {
+ this.region = region;
+ }
+
+ @JsonValue
+ public String value() {
+ return region.value();
+ }
+
+ @Override
+ public String toString() { return value(); }
+
+ public RegionName getRegion() {
+ return region;
+ }
+
+ @JsonCreator
+ public Region(String region) {
+ this.region = com.yahoo.config.provision.RegionName.from(region);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java
new file mode 100644
index 00000000000..f035e200661
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/configserver/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.configserver;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java
new file mode 100644
index 00000000000..bfc451946f6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostJsonModel.java
@@ -0,0 +1,73 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.cost;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON datamodel for the cost api.
+ *
+ * @author smorgrav
+ */
+public class CostJsonModel {
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Application {
+
+ @JsonProperty
+ public String zone;
+ @JsonProperty
+ public String tenant;
+ @JsonProperty
+ public String app;
+ @JsonProperty
+ public int tco;
+ @JsonProperty
+ public float utilization;
+ @JsonProperty
+ public float waste;
+ @JsonProperty
+ public Map<String, Cluster> cluster;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Cluster {
+
+ @JsonProperty
+ public int count;
+ @JsonProperty
+ public String resource;
+ @JsonProperty
+ public float utilization;
+ @JsonProperty
+ public int tco;
+ @JsonProperty
+ public String flavor;
+ @JsonProperty
+ public int waste;
+ @JsonProperty
+ public String type;
+ @JsonProperty
+ public HardwareResources util;
+ @JsonProperty
+ public HardwareResources usage;
+ @JsonProperty
+ public List<String> hostnames;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class HardwareResources {
+
+ @JsonProperty
+ public float mem;
+ @JsonProperty
+ public float disk;
+ @JsonProperty
+ public float cpu;
+ @JsonProperty("diskbusy")
+ public float diskBusy;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java
new file mode 100644
index 00000000000..3cc6d682f4a
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/CostResource.java
@@ -0,0 +1,41 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.cost;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+
+/**
+ * Cost and Utilization APi for hosted Vespa.
+ *
+ * Used to give insight to PEG and application owners about
+ * TOC and if the application is reasonable scaled.
+ *
+ * @author smorgrav
+ */
+@Path("v1")
+@Produces(MediaType.APPLICATION_JSON)
+public interface CostResource {
+
+ @GET
+ @Path("/analysis/cpu")
+ List<CostJsonModel.Application> getCPUAnalysis();
+
+ @GET
+ @Produces("text/csv")
+ @Path("/csv")
+ String getCSV();
+
+ @GET
+ @Path("/apps")
+ List<CostJsonModel.Application> getApplicationsCost();
+
+ @GET
+ @Path("/apps/{environment}/{region}/{application}")
+ CostJsonModel.Application getApplicationCost(@PathParam("application") String appName,
+ @PathParam("region") String regionName,
+ @PathParam("environment") String envName);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java
new file mode 100644
index 00000000000..8e95bd4f6f1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/cost/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.cost;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java
new file mode 100644
index 00000000000..0a5f2809780
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ApplicationId.java
@@ -0,0 +1,29 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class ApplicationId extends NonDefaultIdentifier {
+
+ public ApplicationId(String id) {
+ super(id);
+ }
+
+ public static boolean isLegal(String id) {
+ return strictPattern.matcher(id).matches();
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ validateNoUpperCase();
+ }
+
+ public static void validate(String id) {
+ if (!isLegal(id)) {
+ throwInvalidId(id, "Must match pattern " + strictPattern, "application");
+ }
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java
new file mode 100644
index 00000000000..eb8b5c5256b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/AthensDomain.java
@@ -0,0 +1,29 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class AthensDomain extends Identifier {
+
+ public AthensDomain(String id) {
+ super(id);
+ }
+
+ public boolean isTopLevelDomain() {
+ return !id().contains(".");
+ }
+
+ public AthensDomain getParent() {
+ return new AthensDomain(id().substring(0, lastDot()));
+ }
+
+ public String getName() {
+ return id().substring(lastDot() + 1);
+ }
+
+ private int lastDot() {
+ return id().lastIndexOf('.');
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java
new file mode 100644
index 00000000000..80fe98a4489
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java
@@ -0,0 +1,69 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import com.yahoo.config.provision.Zone;
+
+/**
+ * Application + zone.
+ *
+ * @author smorgrav
+ * @author bratseth
+ */
+public class DeploymentId {
+
+ private final com.yahoo.config.provision.ApplicationId application;
+ private final Zone zone;
+
+ public DeploymentId(com.yahoo.config.provision.ApplicationId application, Zone zone) {
+ this.application = application;
+ this.zone = zone;
+ }
+
+ public com.yahoo.config.provision.ApplicationId applicationId() {
+ return application;
+ }
+ public Zone zone() { return zone; }
+
+
+ public String dottedString() {
+ return unCapitalize(applicationId().tenant().value()) + "."
+ + unCapitalize(applicationId().application().value()) + "."
+ + unCapitalize(zone.environment().value()) + "."
+ + unCapitalize(zone.region().value()) + "."
+ + unCapitalize(application.instance().value());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ DeploymentId other = (DeploymentId) o;
+ if ( ! this.application.equals(other.application)) return false;
+ // TODO: Simplify when Zone implements equals
+ if ( ! this.zone.environment().equals(other.zone.environment())) return false;
+ if ( ! this.zone.region().equals(other.zone.region())) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ // TODO: Simplify when Zone implements hashCode
+ return application.hashCode() +
+ 7 * zone.environment().hashCode() +
+ 31 * zone.region().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return toUserFriendlyString();
+ }
+
+ public String toUserFriendlyString() {
+ return application + " in " + zone;
+ }
+
+ private static String unCapitalize(String str) {
+ return str.toLowerCase().substring(0,1) + str.substring(1);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java
new file mode 100644
index 00000000000..a09e802c251
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/EnvironmentId.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class EnvironmentId extends NonDefaultIdentifier {
+
+ public EnvironmentId(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java
new file mode 100644
index 00000000000..31402825d3c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitBranch.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class GitBranch extends Identifier {
+
+ public GitBranch(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java
new file mode 100644
index 00000000000..289b3ec59a0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitCommit.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class GitCommit extends Identifier {
+
+ public GitCommit(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java
new file mode 100644
index 00000000000..50bbc0bd9f9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/GitRepository.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class GitRepository extends Identifier {
+
+ public GitRepository(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java
new file mode 100644
index 00000000000..3f7437c5d0b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Hostname.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class Hostname extends Identifier {
+
+ public Hostname(String hostname) {
+ super(hostname);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java
new file mode 100644
index 00000000000..70ebc8712d5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Identifier.java
@@ -0,0 +1,103 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+import java.util.Objects;
+import java.util.regex.Pattern;
+
+/**
+ * @author smorgrav
+ */
+public abstract class Identifier {
+
+ protected static final Pattern strictPattern = Pattern.compile("[a-z0-9][a-z0-9-]{0,26}[a-z0-9]");
+ private static final Pattern serializedIdentifierPattern = Pattern.compile("[a-zA-Z0-9_-]+");
+ private static final Pattern serializedPattern = Pattern.compile("[a-zA-Z0-9_.-]+");
+
+ private final String id;
+
+ @JsonCreator
+ public Identifier(String id) {
+ Objects.requireNonNull(id, "Id string cannot be null");
+ this.id = id;
+ validate();
+ }
+
+ public String toDns() {
+ return id.replace('_', '-');
+ }
+
+ @Override
+ public String toString() {
+ return id;
+ }
+
+ @JsonValue
+ public String id() { return id; }
+
+ public String capitalizedType() {
+ String simpleName = this.getClass().getSimpleName();
+ String suffix = "Id";
+ if (simpleName.endsWith(suffix)) {
+ simpleName = simpleName.substring(0, simpleName.length() - suffix.length());
+ }
+ return simpleName;
+ }
+
+ public void validate() {
+ if (id.equals("api")) {
+ throwInvalidId(id, "'api' not allowed.");
+ }
+ }
+
+ protected void validateSerialized() {
+ if (!serializedPattern.matcher(id).matches()) {
+ throwInvalidId(id, "Must match pattern " + serializedPattern);
+ }
+ }
+
+ protected void validateSerializedIdentifier() {
+ if (!serializedIdentifierPattern.matcher(id).matches()) {
+ throwInvalidId(id, "Must match pattern " + serializedIdentifierPattern);
+ }
+ }
+
+ protected void validateNoDefault() {
+ if (id.equals("default")) {
+ throwInvalidId(id, "'default' not allowed.");
+ }
+ }
+
+ protected void validateNoUpperCase() {
+ if (!id.equals(id.toLowerCase()))
+ throwInvalidId(id, "Uppercase not allowed.");
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Identifier identity = (Identifier) o;
+
+ return id.equals(identity.id);
+
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ public static void throwInvalidId(String id, String explanation) {
+ throw new IllegalArgumentException(String.format("Invalid id: %s. %s", id, explanation));
+ }
+
+ public static void throwInvalidId(String id, String explanation, String idName) {
+ throw new IllegalArgumentException(String.format("Invalid %s id: %s. %s", idName, id, explanation));
+ }
+
+}
+
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java
new file mode 100644
index 00000000000..6e3087cdcf6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/InstanceId.java
@@ -0,0 +1,19 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class InstanceId extends SerializedIdentifier {
+
+ public InstanceId(String id) {
+ super(id);
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ validateNoUpperCase();
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java
new file mode 100644
index 00000000000..96f0a9c43f0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/NonDefaultIdentifier.java
@@ -0,0 +1,21 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * TODO: Class description
+ *
+ * @author smorgrav
+ */
+public abstract class NonDefaultIdentifier extends SerializedIdentifier {
+
+ public NonDefaultIdentifier(String id) {
+ super(id);
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ validateNoDefault();
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java
new file mode 100644
index 00000000000..7dde9002310
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/Property.java
@@ -0,0 +1,15 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * A business property.
+ *
+ * @author smorgrav
+ */
+public class Property extends Identifier {
+
+ public Property(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java
new file mode 100644
index 00000000000..c84cfb9b512
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/PropertyId.java
@@ -0,0 +1,29 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import java.util.regex.Pattern;
+
+/**
+ * A business property ID.
+ *
+ * @author frodelu
+ */
+public class PropertyId extends Identifier {
+
+ private static final Pattern PATTERN = Pattern.compile("\\d+");
+
+ public PropertyId(String id) {
+ super(id);
+ }
+
+ /** Returns this id as a long */
+ public long value() { return Long.parseLong(id()); }
+
+ @Override
+ public void validate() {
+ super.validate();
+ if(!PATTERN.matcher(id()).matches()) {
+ throwInvalidId(id(), "Property id must match pattern: " + PATTERN);
+ }
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java
new file mode 100644
index 00000000000..bb6208ff8e3
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RegionId.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class RegionId extends NonDefaultIdentifier {
+
+ public RegionId(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java
new file mode 100644
index 00000000000..11094c69707
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RevisionId.java
@@ -0,0 +1,15 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * An unique identifier of an application package.
+ *
+ * @author smorgrav
+ */
+public class RevisionId extends SerializedIdentifier {
+
+ public RevisionId(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java
new file mode 100644
index 00000000000..aab18595d20
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/RotationId.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class RotationId extends Identifier {
+
+ public RotationId(String id) {
+ super(id);
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ validateSerialized();
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java
new file mode 100644
index 00000000000..b0fb72662c6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ScrewdriverId.java
@@ -0,0 +1,30 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import java.util.regex.Pattern;
+
+/**
+ * @author smorgrav
+ * @author bjorncs
+ */
+public class ScrewdriverId extends Identifier {
+
+ // TODO: If only there was a separate type for this ...
+ // This demonstrates why this subclassing scheme is a bad idea
+ private static final Pattern PATTERN = Pattern.compile("\\d+");
+
+ public ScrewdriverId(String id) {
+ super(id);
+ }
+
+ /** Returns this id as a long */
+ public long value() { return Long.parseLong(id()); }
+
+ @Override
+ public void validate() {
+ super.validate();
+ if(!PATTERN.matcher(id()).matches()) {
+ throwInvalidId(id(), "Screwdriver id must match pattern: " + PATTERN);
+ }
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java
new file mode 100644
index 00000000000..3660262f9c1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/SerializedIdentifier.java
@@ -0,0 +1,22 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * TODO: Class description
+ *
+ * @author smorgrav
+ */
+
+public abstract class SerializedIdentifier extends Identifier {
+
+ public SerializedIdentifier(String id) {
+ super(id);
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ validateSerializedIdentifier();
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java
new file mode 100644
index 00000000000..82cd6d80ec8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java
@@ -0,0 +1,34 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class TenantId extends NonDefaultIdentifier {
+
+ public TenantId(String id) {
+ super(id);
+ }
+
+ public boolean isUser() {
+ return id().startsWith("by-");
+ }
+
+ @Override
+ public void validate() {
+ super.validate();
+ validateNoUpperCase();
+ }
+
+ public static void validate(String id) {
+ if (!strictPattern.matcher(id).matches()) {
+ throwInvalidId(id, "Must match pattern " + strictPattern, "tenant");
+ }
+ }
+
+ /** Return true if this is the user tenant of the given user */
+ public boolean isTenantFor(UserId userId) {
+ return id().equals("by-" + userId.id().replace('_', '-'));
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java
new file mode 100644
index 00000000000..b6b0379bc90
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserGroup.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class UserGroup extends Identifier {
+
+ public UserGroup(String id) {
+ super(id);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java
new file mode 100644
index 00000000000..d2effc76827
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class UserId extends NonDefaultIdentifier {
+
+ public UserId(String id) {
+ super(id);
+ }
+
+ public TenantId toTenantId() {
+ return new TenantId("by-" + id().replace('_', '-'));
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java
new file mode 100644
index 00000000000..79210143d19
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/ZoneId.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+/**
+ * @author smorgrav
+ */
+public class ZoneId extends Identifier {
+
+ public ZoneId(EnvironmentId envId, RegionId regionId) {
+ super(envId.id() + ":" + regionId.id());
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java
new file mode 100644
index 00000000000..211a2ab7fc0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
new file mode 100644
index 00000000000..bbd15707cde
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration;
+
+/**
+ * @author jvenstad
+ */
+public interface BuildService {
+
+ /**
+ * Enqueue a job defined by "buildJob in an external build system, and return the outcome of the enqueue request.
+ * This method should return @false only when a retry is in order, and @true otherwise, e.g., on succes, or for invalid jobs.
+ */
+ boolean trigger(BuildJob buildJob);
+
+ class BuildJob {
+
+ private final long projectId;
+ private final String jobName;
+
+ public BuildJob(long projectId, String jobName) {
+ this.projectId = projectId;
+ this.jobName = jobName;
+ }
+
+ public long projectId() { return projectId; }
+ public String jobName() { return jobName; }
+
+ @Override
+ public String toString() { return jobName + "@" + projectId; }
+
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java
new file mode 100644
index 00000000000..329483a85c5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Contacts.java
@@ -0,0 +1,136 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.Objects;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.unknown;
+import static java.util.Comparator.reverseOrder;
+
+/**
+ * @author jvenstad
+ */
+public interface Contacts {
+
+ /**
+ * Returns the most relevant user of lowest non-empty level above that of @assignee, or, if no such user exists,
+ * the @assignee with @Category information.
+ */
+ static UserContact escalationTargetFrom(Collection<UserContact> userContacts, String assignee) {
+ return userContacts.stream()
+ .filter(contact -> ! contact.username().isEmpty()) // don't assign to empty names
+ .sorted(reverseOrder()).distinct() // Pick out the highest category per user.
+ // Keep the assignee, or the last user on the first non-empty level above her.
+ .sorted().reduce(new UserContact(assignee, assignee, unknown), (current, next) ->
+ next.is(assignee) || (current.is(assignee) ^ current.level() == next.level()) ? next : current);
+ }
+
+ /**
+ * Return a list of all contact entries for property with id @propertyId, where username is set.
+ */
+ Collection<UserContact> userContactsFor(long propertyId);
+
+ /** Returns the URL listing contacts for the given property */
+ URI contactsUri(long propertyId);
+
+ /**
+ * Return a target of escalation above @assignee, from the set of @UserContact entries found for @propertyId.
+ */
+ default UserContact escalationTargetFor(long propertyId, String assignee) {
+ return escalationTargetFrom(userContactsFor(propertyId), assignee);
+ }
+
+ /**
+ * A list of contact roles, in the order in which we look for escalation targets.
+ * Categories must be listed in increasing order of relevancy per level, and by increasing level.
+ */
+ enum Category {
+
+ unknown(-1, Level.none, "Unknown"),
+ admin(54, Level.grunt, "Administrator"), // TODO: Find more grunts?
+ businessOwner(567, Level.owner, "Business Owner"),
+ serviceOwner(646, Level.owner, "Service Engineering Owner"),
+ engineeringOwner(566, Level.owner, "Engineering Owner"),
+ vpBusiness(11, Level.VP, "VP Business"),
+ vpService(647, Level.VP, "VP Service Engineering"),
+ vpEngineering(9, Level.VP, "VP Engineering");
+
+ public final long id;
+ public final Level level;
+ public final String name;
+
+ Category(long id, Level level, String name) {
+ this.id = id;
+ this.level = level;
+ this.name = name;
+ }
+
+ /** Find the category for the given id, or unknown if the id is unknown. */
+ public static Category of(Long id) {
+ for (Category category : values())
+ if (category.id == id)
+ return category;
+ return unknown;
+ }
+
+ public enum Level {
+ none,
+ grunt,
+ owner,
+ VP;
+ }
+
+ }
+
+ /** Container class for user contact information; sorts by category and identifies by username. Immutable. */
+ class UserContact implements Comparable<UserContact> {
+
+ private final String username;
+ private final String name;
+ private final Category category;
+
+ public UserContact(String username, String name, Category category) {
+ Objects.requireNonNull(username, "username cannot be null");
+ Objects.requireNonNull(name, "name cannot be null");
+ Objects.requireNonNull(category, "category cannot be null");
+ this.username = username;
+ this.name = name;
+ this.category = category;
+ }
+
+ public String username() { return username; }
+ public String name() { return name; }
+ public Category category() { return category; }
+ public Category.Level level() { return category.level; }
+
+ public boolean is(String username) { return this.username.equals(username); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UserContact that = (UserContact) o;
+ return Objects.equals(username, that.username);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(username);
+ }
+
+ @Override
+ public int compareTo(@NotNull UserContact other) {
+ return category().compareTo(other.category());
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s, %s, %s", username, name, category.name);
+ }
+
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java
new file mode 100644
index 00000000000..6b7464b9ed0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Issues.java
@@ -0,0 +1,182 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * @author jvenstad
+ */
+public interface Issues {
+
+ /**
+ * Returns information about an issue.
+ * If this issue does not exist this returns an issue id containing the id and default values.
+ */
+ IssueInfo fetch(String issueId);
+
+ /**
+ * Returns the @Meta of all unresolved issues which have the same summary (and queue, if present) as @issue.
+ */
+ List<IssueInfo> fetchSimilarTo(Issue issue);
+
+ /**
+ * Files the given issue
+ *
+ * @return the id of the created issue
+ */
+ String file(Issue issue);
+
+ /**
+ * Update the description fields of the issue stored with id @issueId to be @description.
+ */
+ void update(String issueId, String description);
+
+ /**
+ * Set the assignee of the issue with id @issueId to the user with usename @assignee.
+ */
+ void reassign(String issueId, String assignee);
+
+ /**
+ * Add the user with username @watcher to the watcher list of the issue with id @issueId.
+ */
+ void addWatcher(String issueId, String watcher);
+
+ /**
+ * Post @comment as a comment to the issue with id @issueId.
+ */
+ void comment(String issueId, String comment);
+
+
+ /** Contains information used to file an issue with the responsible party; only @queue is mandatory. */
+ class Classification {
+
+ private final String queue;
+ private final String component;
+ private final String label;
+
+ public Classification(String queue, String component, String label) {
+ if (queue.isEmpty()) throw new IllegalArgumentException("Queue can not be empty!");
+
+ this.queue = queue;
+ this.component = component;
+ this.label = label;
+ }
+
+ public Classification(String queue) {
+ this(queue, null, null);
+ }
+
+ public Classification withComponent(String component) { return new Classification(queue, component, label); }
+ public Classification withLabel(String label) { return new Classification(queue, component, label); }
+
+ public String queue() { return queue; }
+ public Optional<String> component() { return Optional.ofNullable(component); }
+ public Optional<String> label() { return Optional.ofNullable(label); }
+
+ @Override
+ public String toString() {
+ return
+ "Queue : " + queue() + "\n" +
+ "Component : " + component() + "\n" +
+ "Label : " + label() + "\n";
+ }
+
+ }
+
+
+ /** Information about a stored issue */
+ class IssueInfo {
+
+ private final String id;
+ private final String key;
+ private final Instant updated;
+ private final Optional<String> assignee;
+ private final Status status;
+
+ public IssueInfo(String id, String key, Instant updated, Optional<String> assignee, Status status) {
+ if (assignee == null || assignee.isPresent() && assignee.get().isEmpty()) // TODO: Throw on these things
+ assignee = Optional.empty();
+ this.id = id;
+ this.key = key;
+ this.updated = updated;
+ this.assignee = assignee;
+ this.status = status;
+ }
+
+ public IssueInfo withAssignee(Optional<String> assignee) {
+ return new IssueInfo(id, key, updated, assignee, status);
+ }
+
+ public String id() { return id; }
+ public String key() { return key; }
+ public Instant updated() { return updated; }
+ public Optional<String> assignee() { return assignee; }
+ public Status status() { return status; }
+
+ public enum Status {
+
+ toDo("To Do"),
+ inProgress("In Progress"),
+ done("Done"),
+ noCategory("No Category");
+
+ private final String value;
+
+ Status(String value) { this.value = value; }
+
+ public static Status fromValue(String value) {
+ for (Status status : Status.values())
+ if (status.value.equals(value))
+ return status;
+ throw new IllegalArgumentException(value + " is not a valid status.");
+ }
+
+ }
+
+ }
+
+
+ /**
+ * A representation of an issue with a Vespa application which can be reported and escalated through an external issue service.
+ * This class is immutable.
+ *
+ * @author jvenstad
+ */
+ class Issue {
+
+ private final String summary;
+ private final String description;
+ private final Classification classification;
+
+ public Issue(String summary, String description, Classification classification) {
+ if (summary.isEmpty()) throw new IllegalArgumentException("Summary can not be empty.");
+ if (description.isEmpty()) throw new IllegalArgumentException("Description can not be empty.");
+
+ this.summary = summary;
+ this.description = description;
+ this.classification = classification;
+ }
+
+ public Issue(String summary, String description) {
+ this(summary, description, null);
+ }
+
+ public Issue with(Classification classification) {
+ return new Issue(summary, description, classification);
+ }
+ public Issue withDescription(String description) { return new Issue(summary, description, classification); }
+
+ /** Return a new @Issue with the description of @this, but with @appendage appended. */
+ public Issue append(String appendage) {
+ return new Issue(summary, description + "\n\n" + appendage, classification);
+ }
+
+ public String summary() { return summary; }
+ public String description() { return description; }
+ public Optional<Classification> classification() { return Optional.ofNullable(classification); }
+
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java
new file mode 100644
index 00000000000..2068bc7e92d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java
@@ -0,0 +1,66 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+
+/**
+ * A service which returns metric values on request
+ *
+ * @author bratseth
+ */
+public interface MetricsService {
+
+ ApplicationMetrics getApplicationMetrics(ApplicationId application);
+
+ DeploymentMetrics getDeploymentMetrics(ApplicationId application, Zone zone);
+
+ class DeploymentMetrics {
+
+ private final double queriesPerSecond;
+ private final double writesPerSecond;
+ private final long documentCount;
+ private final double queryLatencyMillis;
+ private final double writeLatencyMillis;
+
+ public DeploymentMetrics(double queriesPerSecond, double writesPerSecond,
+ long documentCount,
+ double queryLatencyMillis, double writeLatencyMillis) {
+ this.queriesPerSecond = queriesPerSecond;
+ this.writesPerSecond = writesPerSecond;
+ this.documentCount = documentCount;
+ this.queryLatencyMillis = queryLatencyMillis;
+ this.writeLatencyMillis = writeLatencyMillis;
+ }
+
+ public double queriesPerSecond() { return queriesPerSecond; }
+
+ public double writesPerSecond() { return writesPerSecond; }
+
+ public long documentCount() { return documentCount; }
+
+ public double queryLatencyMillis() { return queryLatencyMillis; }
+
+ public double writeLatencyMillis() { return writeLatencyMillis; }
+
+ }
+
+ class ApplicationMetrics {
+
+ private final double queryServiceQuality;
+ private final double writeServiceQuality;
+
+ public ApplicationMetrics(double queryServiceQuality, double writeServiceQuality) {
+ this.queryServiceQuality = queryServiceQuality;
+ this.writeServiceQuality = writeServiceQuality;
+ }
+
+ /** Returns the quality of service for queries as a number between 1 (perfect) and 0 (none) */
+ public double queryServiceQuality() { return queryServiceQuality; }
+
+ /** Returns the quality of service for writes as a number between 1 (perfect) and 0 (none) */
+ public double writeServiceQuality() { return writeServiceQuality; }
+
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java
new file mode 100644
index 00000000000..652b5495bc5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/Properties.java
@@ -0,0 +1,16 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration;
+
+import java.util.Optional;
+
+/**
+ * @author jvenstad
+ */
+public interface Properties {
+
+ /**
+ * Return the @Issues.Classification listed for the property with id @propertyId.
+ */
+ Optional<Issues.Classification> classificationFor(long propertyId);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java
new file mode 100644
index 00000000000..cb5731164c8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ApplicationAction.java
@@ -0,0 +1,17 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+/**
+ * @author mpolden
+ */
+public enum ApplicationAction {
+ deploy("deployer"),
+ read("reader"),
+ write("writer");
+
+ public final String roleName;
+
+ ApplicationAction(String roleName) {
+ this.roleName = roleName;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java
new file mode 100644
index 00000000000..c1f72fa4370
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/Athens.java
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+/**
+ * Interface for integrating controller with Athens.
+ *
+ * @author mpolden
+ */
+public interface Athens {
+
+ String principalTokenHeader();
+ AthensPrincipal principalFrom(ScrewdriverId screwdriverId);
+ AthensPrincipal principalFrom(UserId userId);
+ NTokenValidator validator();
+ NToken nTokenFrom(String rawToken);
+ UnauthorizedZmsClient unauthorizedZmsClient();
+ ZmsClientFactory zmsClientFactory();
+ AthensDomain screwdriverDomain();
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java
new file mode 100644
index 00000000000..58b878870b9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPrincipal.java
@@ -0,0 +1,59 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+import java.security.Principal;
+import java.util.Objects;
+
+/**
+ * @author bjorncs
+ */
+public class AthensPrincipal implements Principal {
+
+ private final AthensDomain domain;
+ private final UserId userId;
+
+ public AthensPrincipal(AthensDomain domain, UserId userId) {
+ this.domain = domain;
+ this.userId = userId;
+ }
+
+ public UserId getUserId() {
+ return userId;
+ }
+
+ public AthensDomain getDomain() {
+ return domain;
+ }
+
+ public String toYRN() {
+ return domain.id() + "." + userId.id();
+ }
+
+ @Override
+ public String toString() {
+ return toYRN();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AthensPrincipal that = (AthensPrincipal) o;
+ return Objects.equals(domain, that.domain) &&
+ Objects.equals(userId, that.userId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(domain, userId);
+ }
+
+ @Override
+ public String getName() {
+ return userId.id();
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java
new file mode 100644
index 00000000000..9bbb5f28d8f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensPublicKey.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import java.security.PublicKey;
+import java.util.Objects;
+
+/**
+ * @author bjorncs
+ */
+public class AthensPublicKey {
+ private final PublicKey publicKey;
+ private final String keyId;
+
+ public AthensPublicKey(PublicKey publicKey, String keyId) {
+ this.publicKey = publicKey;
+ this.keyId = keyId;
+ }
+
+ public PublicKey getPublicKey() {
+ return publicKey;
+ }
+
+ public String getKeyId() {
+ return keyId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AthensPublicKey that = (AthensPublicKey) o;
+ return Objects.equals(publicKey, that.publicKey) &&
+ Objects.equals(keyId, that.keyId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(publicKey, keyId);
+ }
+
+ @Override
+ public String toString() {
+ return "AthensPublicKey{" +
+ "publicKey=" + publicKey +
+ ", keyId='" + keyId + '\'' +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java
new file mode 100644
index 00000000000..42af966be3d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/AthensService.java
@@ -0,0 +1,51 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+
+import java.util.Objects;
+
+/**
+ * @author bjorncs
+ */
+public class AthensService {
+
+ private final AthensDomain domain;
+ private final String serviceName;
+
+ public AthensService(AthensDomain domain, String serviceName) {
+ this.domain = domain;
+ this.serviceName = serviceName;
+ }
+
+ public String toFullServiceName() {
+ return domain.id() + "." + serviceName;
+ }
+
+ public AthensDomain getDomain() {
+ return domain;
+ }
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AthensService that = (AthensService) o;
+ return Objects.equals(domain, that.domain) &&
+ Objects.equals(serviceName, that.serviceName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(domain, serviceName);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("AthensService(%s)", toFullServiceName());
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java
new file mode 100644
index 00000000000..9c21d5814cb
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/InvalidTokenException.java
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+/**
+ * @author bjorncs
+ */
+public class InvalidTokenException extends Exception {
+ public InvalidTokenException(String message) {
+ super(message);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java
new file mode 100644
index 00000000000..b74872b4c6a
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NToken.java
@@ -0,0 +1,21 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+import java.security.PublicKey;
+
+/**
+ * @author mpolden
+ */
+public interface NToken {
+
+ AthensPrincipal getPrincipal();
+ UserId getUser();
+ AthensDomain getDomain();
+ String getToken();
+ String getKeyId();
+ void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException;
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java
new file mode 100644
index 00000000000..905d7d864a3
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/NTokenValidator.java
@@ -0,0 +1,12 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+/**
+ * @author mpolden
+ */
+public interface NTokenValidator {
+
+ void preloadPublicKeys();
+ AthensPrincipal validate(NToken nToken) throws InvalidTokenException;
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java
new file mode 100644
index 00000000000..d1996bdbd45
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/UnauthorizedZmsClient.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+
+import java.util.List;
+
+/**
+ * @author gv
+ */
+public class UnauthorizedZmsClient {
+
+ private final ZmsClient client;
+
+ public UnauthorizedZmsClient(ZmsClientFactory zmsClientFactory) {
+ client = zmsClientFactory.createClientWithoutPrincipal();
+ }
+
+ public List<AthensDomain> getDomainList(String prefix) {
+ return client.getDomainList(prefix);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java
new file mode 100644
index 00000000000..7ff54957e16
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClient.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+
+import java.util.List;
+
+/**
+ * @author bjorncs
+ */
+public interface ZmsClient {
+ void createTenant(AthensDomain tenantDomain);
+
+ void deleteTenant(AthensDomain tenantDomain);
+
+ void addApplication(AthensDomain tenantDomain, ApplicationId applicationName);
+
+ void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName);
+
+ boolean hasApplicationAccess(AthensPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName);
+
+ boolean hasTenantAdminAccess(AthensPrincipal principal, AthensDomain tenantDomain);
+
+ // Used before vespa tenancy is established for the domain.
+ boolean isDomainAdmin(AthensPrincipal principal, AthensDomain domain);
+
+ List<AthensDomain> getDomainList(String prefix);
+
+ List<AthensDomain> getTenantDomainsForUser(AthensPrincipal principal);
+
+ AthensPublicKey getPublicKey(AthensService service, String keyId);
+
+ List<AthensPublicKey> getPublicKeys(AthensService service);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java
new file mode 100644
index 00000000000..24a2d67ebf6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsClientFactory.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+/**
+ * @author bjorncs
+ */
+public interface ZmsClientFactory {
+ ZmsClient createClientWithServicePrincipal();
+
+ ZmsClient createClientWithAuthorizedServiceToken(NToken authorizedServiceToken);
+
+ ZmsClient createClientWithoutPrincipal();
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java
new file mode 100644
index 00000000000..ed5b2daca86
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsException.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+/**
+ * @author bjorncs
+ */
+public class ZmsException extends RuntimeException {
+
+ private final int code;
+
+ public ZmsException(Throwable t, int code) {
+ super(t.getMessage(), t);
+ this.code = code;
+ }
+
+ public ZmsException(int code) {
+ this.code = code;
+ }
+
+ public int getCode() {
+ return code;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java
new file mode 100644
index 00000000000..4f8e5f5ff05
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/ZmsKeystore.java
@@ -0,0 +1,19 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import java.security.PublicKey;
+import java.util.Optional;
+
+/**
+ * Interface for a keystore containing public keys for Athens services
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+public interface ZmsKeystore {
+ Optional<PublicKey> getPublicKey(AthensService service, String keyId);
+
+ default void preloadKeys(AthensService service) {
+ // Default implementation is noop
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java
new file mode 100644
index 00000000000..8a02d0dcff5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensDbMock.java
@@ -0,0 +1,73 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens.mock;
+
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author bjorncs
+ */
+public class AthensDbMock {
+
+ public final Map<AthensDomain, Domain> domains = new HashMap<>();
+
+ public AthensDbMock addDomain(Domain domain) {
+ domains.put(domain.name, domain);
+ return this;
+ }
+
+ public static class Domain {
+
+ public final AthensDomain name;
+ public final Set<AthensPrincipal> admins = new HashSet<>();
+ public final Set<AthensPrincipal> tenantAdmins = new HashSet<>();
+ public final Map<ApplicationId, Application> applications = new HashMap<>();
+ public boolean isVespaTenant = false;
+
+ public Domain(AthensDomain name) {
+ this.name = name;
+ }
+
+ public Domain admin(AthensPrincipal user) {
+ admins.add(user);
+ return this;
+ }
+
+ public Domain tenantAdmin(AthensPrincipal user) {
+ tenantAdmins.add(user);
+ return this;
+ }
+
+ /**
+ * Simulates establishing Vespa tenancy in Athens.
+ */
+ public void markAsVespaTenant() {
+ isVespaTenant = true;
+ }
+
+ }
+
+ public static class Application {
+
+ public final Map<ApplicationAction, Set<AthensPrincipal>> acl = new HashMap<>();
+
+ public Application() {
+ acl.put(ApplicationAction.deploy, new HashSet<>());
+ acl.put(ApplicationAction.read, new HashSet<>());
+ acl.put(ApplicationAction.write, new HashSet<>());
+ }
+
+ public Application addRoleMember(ApplicationAction action, AthensPrincipal user) {
+ acl.get(action).add(user);
+ return this;
+ }
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java
new file mode 100644
index 00000000000..a993c6e3da3
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/AthensMock.java
@@ -0,0 +1,95 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens.mock;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.InvalidTokenException;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NTokenValidator;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.UnauthorizedZmsClient;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory;
+
+/**
+ * @author mpolden
+ */
+public class AthensMock extends AbstractComponent implements Athens {
+
+ private static final AthensDomain userDomain = new AthensDomain("domain1");
+ private static final AthensDomain screwdriverDomain = new AthensDomain("screwdriver-domain");
+
+ private final ZmsClientFactory zmsClientFactory;
+ private final UnauthorizedZmsClient unauthorizedZmsClient;
+ private final NTokenValidator nTokenValidator;
+
+ public AthensMock(AthensDbMock athensDb, NTokenValidator nTokenValidator) {
+ this.zmsClientFactory = new ZmsClientFactoryMock(athensDb);
+ this.unauthorizedZmsClient = new UnauthorizedZmsClient(zmsClientFactory);
+ this.nTokenValidator = nTokenValidator;
+ }
+
+ public AthensMock(AthensDbMock athensDbMock) {
+ this(athensDbMock, mockValidator);
+ }
+
+ @Inject
+ public AthensMock() {
+ this(new AthensDbMock(), mockValidator);
+ }
+
+ @Override
+ public String principalTokenHeader() {
+ return "X-Athens-Token";
+ }
+
+ @Override
+ public AthensPrincipal principalFrom(ScrewdriverId screwdriverId) {
+ return new AthensPrincipal(screwdriverDomain, new UserId("screwdriver-" + screwdriverId.id()));
+ }
+
+ @Override
+ public AthensPrincipal principalFrom(UserId userId) {
+ return new AthensPrincipal(userDomain, userId);
+ }
+
+ @Override
+ public NTokenValidator validator() {
+ return nTokenValidator;
+ }
+
+ @Override
+ public NToken nTokenFrom(String rawToken) {
+ return new NTokenMock(rawToken);
+ }
+
+ @Override
+ public UnauthorizedZmsClient unauthorizedZmsClient() {
+ return unauthorizedZmsClient;
+ }
+
+ @Override
+ public ZmsClientFactory zmsClientFactory() {
+ return zmsClientFactory;
+ }
+
+ @Override
+ public AthensDomain screwdriverDomain() {
+ return screwdriverDomain;
+ }
+
+ private static final NTokenValidator mockValidator = new NTokenValidator() {
+ @Override
+ public void preloadPublicKeys() {
+ }
+
+ @Override
+ public AthensPrincipal validate(NToken nToken) throws InvalidTokenException {
+ return nToken.getPrincipal();
+ }
+ };
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java
new file mode 100644
index 00000000000..ae23a69e409
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/NTokenMock.java
@@ -0,0 +1,68 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens.mock;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.InvalidTokenException;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+
+import java.security.PublicKey;
+import java.util.Objects;
+
+/**
+ * @author mpolden
+ */
+public class NTokenMock implements NToken {
+
+ private static final AthensDomain domain = new AthensDomain("test");
+ private static final UserId userId = new UserId("user");
+
+ private final String rawToken;
+
+ public NTokenMock(String rawToken) {
+ this.rawToken = rawToken;
+ }
+
+ @Override
+ public AthensPrincipal getPrincipal() {
+ return new AthensPrincipal(domain, userId);
+ }
+
+ @Override
+ public UserId getUser() {
+ return userId;
+ }
+
+ @Override
+ public AthensDomain getDomain() {
+ return domain;
+ }
+
+ @Override
+ public String getToken() {
+ return "test-token";
+ }
+
+ @Override
+ public String getKeyId() {
+ return "test-key";
+ }
+
+ @Override
+ public void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException {
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof NTokenMock)) return false;
+ NTokenMock that = (NTokenMock) o;
+ return Objects.equals(rawToken, that.rawToken);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rawToken);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java
new file mode 100644
index 00000000000..73d971a27fe
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientFactoryMock.java
@@ -0,0 +1,55 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens.mock;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author bjorncs
+ */
+public class ZmsClientFactoryMock extends AbstractComponent implements ZmsClientFactory {
+
+ private static final Logger log = Logger.getLogger(ZmsClientFactoryMock.class.getName());
+
+ private final AthensDbMock athens;
+
+ public ZmsClientFactoryMock() {
+ this(new AthensDbMock());
+ }
+
+ ZmsClientFactoryMock(AthensDbMock athens) {
+ this.athens = athens;
+ }
+
+ public AthensDbMock getSetup() {
+ return athens;
+ }
+
+ @Override
+ public ZmsClient createClientWithServicePrincipal() {
+ log("createClientWithServicePrincipal()");
+ return new ZmsClientMock(athens);
+ }
+
+ @Override
+ public ZmsClient createClientWithAuthorizedServiceToken(NToken authorizedServiceToken) {
+ log("createClientWithAuthorizedServiceToken(authorizedServiceToken='%s')", authorizedServiceToken);
+ return new ZmsClientMock(athens);
+ }
+
+ @Override
+ public ZmsClient createClientWithoutPrincipal() {
+ log("createClientWithoutPrincipal()");
+ return new ZmsClientMock(athens);
+ }
+
+ private static void log(String format, Object... args) {
+ log.log(Level.INFO, String.format(format, args));
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java
new file mode 100644
index 00000000000..97f391f792d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/ZmsClientMock.java
@@ -0,0 +1,131 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.athens.mock;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPublicKey;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensService;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * @author bjorncs
+ */
+public class ZmsClientMock implements ZmsClient {
+
+ private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName());
+
+ private final AthensDbMock athens;
+
+ public ZmsClientMock(AthensDbMock athens) {
+ this.athens = athens;
+ }
+
+ @Override
+ public void createTenant(AthensDomain tenantDomain) {
+ log("createTenant(tenantDomain='%s')", tenantDomain);
+ getDomainOrThrow(tenantDomain, false).isVespaTenant = true;
+ }
+
+ @Override
+ public void deleteTenant(AthensDomain tenantDomain) {
+ log("deleteTenant(tenantDomain='%s')", tenantDomain);
+ AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, false);
+ domain.isVespaTenant = false;
+ domain.applications.clear();
+ domain.tenantAdmins.clear();
+ }
+
+ @Override
+ public void addApplication(AthensDomain tenantDomain, ApplicationId applicationName) {
+ log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName);
+ AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, true);
+ if (!domain.applications.containsKey(applicationName)) {
+ domain.applications.put(applicationName, new AthensDbMock.Application());
+ }
+ }
+
+ @Override
+ public void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName) {
+ log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName);
+ getDomainOrThrow(tenantDomain, true).applications.remove(applicationName);
+ }
+
+ @Override
+ public boolean hasApplicationAccess(AthensPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName) {
+ log("hasApplicationAccess(principal='%s', action='%s', tenantDomain='%s', applicationName='%s')",
+ principal, action, tenantDomain, applicationName);
+ AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, true);
+ AthensDbMock.Application application = domain.applications.get(applicationName);
+ if (application == null) {
+ throw zmsException(400, "Application '%s' not found", applicationName);
+ }
+ return domain.admins.contains(principal) || application.acl.get(action).contains(principal);
+ }
+
+ @Override
+ public boolean hasTenantAdminAccess(AthensPrincipal principal, AthensDomain tenantDomain) {
+ log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", principal, tenantDomain);
+ return isDomainAdmin(principal, tenantDomain) ||
+ getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(principal);
+ }
+
+ @Override
+ public boolean isDomainAdmin(AthensPrincipal principal, AthensDomain domain) {
+ log("isDomainAdmin(principal='%s', domain='%s')", principal, domain);
+ return getDomainOrThrow(domain, false).admins.contains(principal);
+ }
+
+ @Override
+ public List<AthensDomain> getDomainList(String prefix) {
+ log("getDomainList()");
+ return new ArrayList<>(athens.domains.keySet());
+ }
+
+ @Override
+ public List<AthensDomain> getTenantDomainsForUser(AthensPrincipal principal) {
+ log("getTenantDomainsForUser(principal='%s')", principal);
+ return athens.domains.values().stream()
+ .filter(domain -> domain.tenantAdmins.contains(principal) || domain.admins.contains(principal))
+ .map(domain -> domain.name)
+ .collect(toList());
+ }
+
+ @Override
+ public AthensPublicKey getPublicKey(AthensService service, String keyId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<AthensPublicKey> getPublicKeys(AthensService service) {
+ throw new UnsupportedOperationException();
+ }
+
+ private AthensDbMock.Domain getDomainOrThrow(AthensDomain domainName, boolean verifyVespaTenant) {
+ AthensDbMock.Domain domain = Optional.ofNullable(athens.domains.get(domainName))
+ .orElseThrow(() -> zmsException(400, "Domain '%s' not found", domainName));
+ if (verifyVespaTenant && !domain.isVespaTenant) {
+ throw zmsException(400, "Domain not a Vespa tenant: '%s'", domainName);
+ }
+ return domain;
+ }
+
+ private static ZmsException zmsException(int code, String message, Object... args) {
+ return new ZmsException(new RuntimeException(String.format(message, args)), code);
+ }
+
+ private static void log(String format, Object... args) {
+ log.log(Level.INFO, String.format(format, args));
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java
new file mode 100644
index 00000000000..d4454503786
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/mock/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.athens.mock;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java
new file mode 100644
index 00000000000..eabe214abf2
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athens/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.athens;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java
new file mode 100644
index 00000000000..87970458855
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/AttributeMapping.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author mortent
+ */
+public class AttributeMapping {
+
+ private final String attribute;
+ private final List<String> chefPath;
+
+ private AttributeMapping(String attribute, List<String> chefPath) {
+ this.chefPath = chefPath;
+ this.attribute = attribute;
+ }
+
+ public static AttributeMapping simpleMapping(String attribute) {
+ return new AttributeMapping(attribute, Collections.singletonList(attribute));
+ }
+
+ public static AttributeMapping deepMapping(String attribute, List<String> chefPath) {
+ return new AttributeMapping(attribute, chefPath);
+ }
+
+ public String toString() {
+ return String.format("\"%s\": [%s]", attribute,
+ chefPath.stream().map(s -> String.format("\"%s\"", s))
+ .collect(Collectors.joining(","))
+ );
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java
new file mode 100644
index 00000000000..693947b6f61
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/Chef.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef;
+
+
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult;
+
+import java.net.URL;
+import java.util.List;
+
+public interface Chef {
+
+ ChefResource getApi();
+
+ ChefNode getNode(String name);
+
+ Client getClient(String name);
+
+ ChefNode deleteNode(String name);
+
+ Client deleteClient(String name);
+
+ NodeResult searchNodeByFQDN(String fqdn);
+
+ NodeResult searchNodes(String query);
+
+ PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> attributeMappings);
+
+ void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName);
+
+ ChefEnvironment getChefEnvironment(String environmentName);
+
+ CookBook getCookbook(String cookbookName, String cookbookVersion);
+
+ String downloadResource(URL resourceURL);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java
new file mode 100644
index 00000000000..1b2dad34b8d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/ChefMock.java
@@ -0,0 +1,112 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef;
+
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefEnvironment;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefNode;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.ChefResource;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.Client;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.CookBook;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.NodeResult;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult;
+
+import javax.ws.rs.NotFoundException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author mpolden
+ */
+public class ChefMock implements Chef {
+
+ private final NodeResult result;
+ private final List<String> chefEnvironments;
+
+ public ChefMock() {
+ result = new NodeResult();
+ result.rows = new ArrayList<>();
+ chefEnvironments = new ArrayList<>();
+ chefEnvironments.add("hosted-verified-prod");
+ chefEnvironments.add("hosted-infra-cd");
+ }
+
+ @Override
+ public ChefResource getApi() {
+ return null;
+ }
+
+ @Override
+ public ChefNode getNode(String name) {
+ return null;
+ }
+
+ @Override
+ public Client getClient(String name) {
+ return null;
+ }
+
+ @Override
+ public ChefNode deleteNode(String name) {
+ return null;
+ }
+
+ @Override
+ public Client deleteClient(String name) {
+ return null;
+ }
+
+ public void addSearchResult(ChefNode node) {
+ result.rows.add(node);
+ }
+
+ @Override
+ public NodeResult searchNodeByFQDN(String fqdn) {
+ return result;
+ }
+
+ @Override
+ public NodeResult searchNodes(String query) {
+ return result;
+ }
+
+ @Override
+ public PartialNodeResult partialSearchNodes(String query, List<AttributeMapping> returnAttributes) {
+ PartialNodeResult partialNodeResult = new PartialNodeResult();
+ partialNodeResult.rows = result.rows.stream()
+ .map(chefNode -> {
+ Map<String, String> data = new HashMap<>();
+ data.put("fqdn", chefNode.name);
+ return new PartialNode(data);
+ })
+ .collect(Collectors.toList());
+ return partialNodeResult;
+ }
+
+ @Override
+ public void copyChefEnvironment(String fromEnvironmentName, String toEnvironmentName) {
+ if(!chefEnvironments.contains(fromEnvironmentName)) {
+ throw new NotFoundException(String.format("Source chef environment %s does not exist", fromEnvironmentName));
+ }
+ chefEnvironments.add(toEnvironmentName);
+ }
+
+ @Override
+ public ChefEnvironment getChefEnvironment(String environmentName) {
+ return null;
+ }
+
+ @Override
+ public CookBook getCookbook(String cookbookName, String cookbookVersion) {
+ return null;
+ }
+
+ @Override
+ public String downloadResource(URL resourceURL) {
+ return "";
+ }
+}
+
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java
new file mode 100644
index 00000000000..5d3d4b87b74
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.chef;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java
new file mode 100644
index 00000000000..8576949280b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefEnvironment.java
@@ -0,0 +1,110 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Map;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ChefEnvironment {
+
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("default_attributes")
+ private Map<String, Object> attributes;
+ @JsonProperty("override_attributes")
+ private Map<String, Object> overrideAttributes;
+ @JsonProperty("description")
+ private String description;
+ @JsonProperty("cookbook_versions")
+ private Map<String, String> cookbookVersions;
+
+ // internal
+ @JsonProperty("json_class")
+ private final String _jsonClass = "Chef::Environment";
+ @JsonProperty("chef_type")
+ private final String _chefType = "environment";
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Builder copy() {
+ return builder()
+ .name(name)
+ .attributes(attributes)
+ .overrideAttributes(overrideAttributes)
+ .cookbookVersions(cookbookVersions)
+ .description(description);
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public Map<String, String> getCookbookVersions() {
+ return cookbookVersions;
+ }
+
+ public Map<String, Object> getAttributes() {
+ return attributes;
+ }
+
+ public Map<String, Object> getOverrideAttributes() {
+ return overrideAttributes;
+ }
+
+ public static class Builder {
+ private String name;
+ private Map<String, Object> attributes;
+ private String description;
+ private Map<String, Object> overrideAttributes;
+ private Map<String, String> cookbookVersions;
+
+ public Builder name(String name){
+ this.name = name;
+ return this;
+ }
+
+ public Builder attributes(Map<String, Object> defaultAttributes) {
+ this.attributes = defaultAttributes;
+ return this;
+ }
+
+ public Builder overrideAttributes(Map<String, Object> overrideAttributes) {
+ this.overrideAttributes = overrideAttributes;
+ return this;
+ }
+
+ public Builder cookbookVersions(Map<String, String> cookbookVersions) {
+ this.cookbookVersions = cookbookVersions;
+ return this;
+ }
+
+ public Builder description(String description) {
+ this.description = description;
+ return this;
+ }
+
+ public ChefEnvironment build() {
+ ChefEnvironment chefEnvironment = new ChefEnvironment();
+ chefEnvironment.name = name;
+ chefEnvironment.description = description;
+ chefEnvironment.cookbookVersions = cookbookVersions;
+ chefEnvironment.attributes = attributes;
+ chefEnvironment.overrideAttributes = overrideAttributes;
+
+ return chefEnvironment;
+ }
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java
new file mode 100644
index 00000000000..08d9a1045e8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefNode.java
@@ -0,0 +1,118 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ChefNode {
+
+ @JsonProperty("name")
+ public String name;
+
+ @JsonProperty("chef_environment")
+ public String chefEnvironment;
+
+ @JsonProperty("run_list")
+ public List<String> runList;
+
+ @JsonProperty("json_class")
+ public String jsonClass;
+
+ @JsonProperty("chef_type")
+ public String chefType;
+
+ @JsonProperty("automatic")
+ public Map<String, Object> automaticAttributes;
+
+ @JsonProperty("normal")
+ public Map<String, Object> normalAttributes;
+
+ @JsonProperty("default")
+ public Map<String, Object> defaultAttributes;
+
+ @JsonProperty("override")
+ public Map<String, Object> overrideAttributes;
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static Builder builder(ChefNode src) {
+ return new Builder(src);
+ }
+
+ public static class Builder {
+ private String name;
+ private String chefEnvironment;
+ private List<String> runList;
+ private String jsonClass;
+ private String chefType;
+ private Map<String, Object> automaticAttributes;
+ private Map<String, Object> normalAttributes;
+ private Map<String, Object> defaultAttributes;
+ private Map<String, Object> overrideAttributes;
+
+ private Builder(){}
+
+ private Builder(ChefNode src){
+ this.name = src.name;
+ this.chefEnvironment = src.chefEnvironment;
+ this.runList = new ArrayList<>(src.runList);
+ this.jsonClass = src.jsonClass;
+ this.chefType = src.chefType;
+ this.automaticAttributes = new HashMap<>(src.automaticAttributes);
+ this.normalAttributes = new HashMap<>(src.normalAttributes);
+ this.defaultAttributes = new HashMap<>(src.defaultAttributes);
+ this.overrideAttributes = new HashMap<>(src.overrideAttributes);
+ }
+
+ public Builder name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder chefEnvironment(String chefEnvironment) {
+ this.chefEnvironment = chefEnvironment;
+ return this;
+ }
+
+ public ChefNode build(){
+ ChefNode node = new ChefNode();
+ node.name = this.name;
+ node.chefEnvironment = this.chefEnvironment;
+ node.runList = this.runList;
+ node.jsonClass = this.jsonClass;
+ node.chefType = this.chefType;
+ node.automaticAttributes = this.automaticAttributes;
+ node.overrideAttributes = this.overrideAttributes;
+ node.defaultAttributes = this.defaultAttributes;
+ node.normalAttributes = this.normalAttributes;
+ return node;
+ }
+
+ }
+
+ @Override
+ public String toString() {
+ return "Node{" +
+ "name='" + name + '\'' +
+ ", chefEnvironment='" + chefEnvironment + '\'' +
+ ", runList=" + runList +
+ ", jsonClass='" + jsonClass + '\'' +
+ ", chefType='" + chefType + '\'' +
+ ", automaticAttributes=" + automaticAttributes +
+ ", normalAttributes=" + normalAttributes +
+ ", defaultAttributes=" + defaultAttributes +
+ ", overrideAttributes=" + overrideAttributes +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java
new file mode 100644
index 00000000000..98eeb0770fc
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/ChefResource.java
@@ -0,0 +1,74 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+
+/**
+ * @author mortent
+ * @author mpolden
+ */
+
+@Path("/")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public interface ChefResource {
+
+ @Path("/organizations/{organization}/environments/{environment}/nodes")
+ @Consumes("application/json")
+ @GET
+ List<String> getNodes(@PathParam("organization") String organization, @PathParam("environment") String environment);
+
+ @GET
+ @Path("/organizations/{organization}/nodes/{nodename}")
+ ChefNode getNode(@PathParam("organization") String organization, @PathParam("nodename") String nodename);
+
+ @PUT
+ @Path("/organizations/{organization}/nodes/{nodename}")
+ ChefNode updateNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName, String node);
+
+ @DELETE
+ @Path("/organizations/{organization}/nodes/{nodename}")
+ ChefNode deleteNode(@PathParam("organization") String organization, @PathParam("nodename") String nodeName);
+
+ @GET
+ @Path("/organizations/{organization}/clients/{name}")
+ Client getClient(@PathParam("organization") String organization, @PathParam("name") String name);
+
+ @DELETE
+ @Path("/organizations/{organization}/clients/{name}")
+ Client deleteClient(@PathParam("organization") String organization, @PathParam("name") String name);
+
+ @GET
+ @Path("/organizations/{organization}/environments/{environment}")
+ ChefEnvironment getEnvironment(@PathParam("organization") String organization, @PathParam("environment") String environment);
+
+ @PUT
+ @Path("/organizations/{organization}/environments/{name}")
+ String updateEnvironment(@PathParam("organization") String organization, @PathParam("name") String chefEnvironmentName, String contentAsString);
+
+ @POST
+ @Path("/organizations/{organization}/environments")
+ String createEnvironment(@PathParam("organization") String organization, String contentAsString);
+
+ @GET
+ @Path("/organizations/{organization}/search/node")
+ NodeResult searchNode(@PathParam("organization") String organization, @QueryParam("q") String query);
+
+ @POST
+ @Path("/organizations/{organization}/search/node")
+ PartialNodeResult partialSearchNode(@PathParam("organization") String organization, @QueryParam("q") String query, @QueryParam("rows") int rows, String keys);
+
+ @GET
+ @Path("/organizations/{organization}/cookbooks/{name}/{version}")
+ CookBook getCookBook(@PathParam("organization") String organization, @PathParam("name") String name, @PathParam("version") String version);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java
new file mode 100644
index 00000000000..0ea9b0e9997
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/Client.java
@@ -0,0 +1,25 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author mpolden
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Client {
+
+ @JsonProperty("name")
+ public String name;
+ @JsonProperty("validator")
+ public boolean validator;
+
+ @Override
+ public String toString() {
+ return "Client{" +
+ "name='" + name + '\'' +
+ ", validator=" + validator +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java
new file mode 100644
index 00000000000..ab49ac9ff60
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/CookBook.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class CookBook {
+ public final String name;
+ public final List<Attributes> attributes;
+
+ public CookBook(@JsonProperty("name") String name, @JsonProperty("attributes") List<Attributes> attributes) {
+ this.name = name;
+ this.attributes = attributes;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Attributes {
+ public final String name;
+ public final String url;
+
+ public Attributes(@JsonProperty("name") String name, @JsonProperty("url") String url) {
+ this.name = name;
+ this.url = url;
+ }
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java
new file mode 100644
index 00000000000..e3ab431473f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/NodeResult.java
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author mpolden
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class NodeResult {
+ @JsonProperty("total")
+ public int total;
+ @JsonProperty("start")
+ public int start;
+ @JsonProperty("rows")
+ public List<ChefNode> rows;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java
new file mode 100644
index 00000000000..f4aa90021b1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNode.java
@@ -0,0 +1,40 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PartialNode {
+
+ @JsonProperty("data")
+ private final Map<String, String> data;
+
+ @JsonCreator
+ public PartialNode(@JsonProperty("data") Map<String, String> data) {
+ this.data = data;
+ }
+
+ public Optional<String> getValue(String key) {
+ return Optional.ofNullable(data.get(key));
+ }
+
+ public String getFqdn() {
+ return getValue("fqdn").orElse("");
+ }
+
+ public String getName() {
+ return getValue("name").orElse("");
+ }
+
+ public Double getOhaiTime() {
+ return Double.parseDouble(getValue("ohai_time").orElse("0.0"));
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java
new file mode 100644
index 00000000000..9925237a193
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/PartialNodeResult.java
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PartialNodeResult {
+ @JsonProperty("total")
+ public int total;
+ @JsonProperty("start")
+ public int start;
+ @JsonProperty("rows")
+ public List<PartialNode> rows;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java
new file mode 100644
index 00000000000..7d06571507e
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/chef/rest/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.chef.rest;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java
new file mode 100644
index 00000000000..1958c5bd0ff
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerClient.java
@@ -0,0 +1,69 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * @author Oyvind Grønnesby
+ */
+public interface ConfigServerClient {
+
+ interface PreparedApplication {
+ void activate();
+ List<Log> messages();
+ PrepareResponse prepareResponse();
+ }
+
+ PreparedApplication prepare(DeploymentId applicationInstance, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content);
+
+ List<String> getNodeQueryHost(DeploymentId applicationInstance, String type) throws NoInstanceException;
+
+ void restart(DeploymentId applicationInstance, Optional<Hostname> hostname) throws NoInstanceException;
+
+ void deactivate(DeploymentId applicationInstance) throws NoInstanceException;
+
+ JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds);
+
+ JsonNode grabLog(DeploymentId applicationInstance);
+
+ ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region);
+
+ Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath);
+
+ /** Returns the version this particular config server is running */
+ Version version(URI configserverUri);
+
+ /**
+ * Set new status on en endpoint in one zone.
+ *
+ * @param deployment The application/zone pair
+ * @param endpoint The endpoint to modify
+ * @param status The new status with metadata
+ * @throws IOException If trouble contacting the server
+ */
+ void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status) throws IOException;
+
+ /**
+ * Get the endpoint status for an app in one zone
+ *
+ * @param deployment The application/zone pair
+ * @param endpoint The endpoint to modify
+ * @return The endpoint status with metadata
+ * @throws IOException If trouble contacting the server
+ */
+ EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) throws IOException;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java
new file mode 100644
index 00000000000..f578322ac76
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java
@@ -0,0 +1,41 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+import java.net.URI;
+
+/**
+ * @author Tony Vaagenes
+ */
+public class ConfigServerException extends RuntimeException {
+
+ private final URI serverUri;
+ private final ErrorCode errorCode;
+
+ public ConfigServerException(URI serverUri, String message, ErrorCode errorCode, Throwable cause) {
+ super(message, cause);
+ this.serverUri = serverUri;
+ this.errorCode = errorCode;
+ }
+
+ public ErrorCode getErrorCode() {
+ return errorCode;
+ }
+
+ public URI getServerUri() {
+ return serverUri;
+ }
+
+ // TODO: Copied from Vespa. Expose these in Vespa and use them here
+ public enum ErrorCode {
+ APPLICATION_LOCK_FAILURE,
+ BAD_REQUEST,
+ INTERNAL_SERVER_ERROR,
+ INVALID_APPLICATION_PACKAGE,
+ METHOD_NOT_ALLOWED,
+ NOT_FOUND,
+ OUT_OF_CAPACITY,
+ REQUEST_TIMEOUT,
+ UNKNOWN_VESPA_VERSION
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java
new file mode 100644
index 00000000000..ba5d740d0e1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Log.java
@@ -0,0 +1,15 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * @author Tony Vaagenes
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Log {
+ public long time;
+ public String level;
+ public String message;
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java
new file mode 100644
index 00000000000..a415721407f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NoInstanceException.java
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+/**
+ * @author Tony Vaagenes
+ */
+public class NoInstanceException extends Exception {
+ public NoInstanceException(String msg) {
+ super(msg);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java
new file mode 100644
index 00000000000..6054e05149b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/PrepareResponse.java
@@ -0,0 +1,22 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+
+import java.net.URI;
+import java.util.List;
+
+/**
+ * @author Tony Vaagenes
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PrepareResponse {
+ public TenantId tenant;
+ @JsonProperty("activate") public URI activationUri;
+ public String message;
+ public List<Log> log;
+ public ConfigChangeActions configChangeActions;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java
new file mode 100644
index 00000000000..10eddb2628f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java
new file mode 100644
index 00000000000..9bc9cfa8ed0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ApplicationCost.java
@@ -0,0 +1,105 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.cost;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Cost data model for an application instance. I.e one running vespa application in one zone.
+ *
+ * @author smorgrav
+ */
+// TODO: Make immutable
+// TODO: Make the Application own this and rename to Cost
+// TODO: Enforce constraints
+// TODO: Remove application id elements
+// TODO: Model zone as Zone
+// TODO: Cost per zone + total
+// TODO: Use doubles
+public class ApplicationCost {
+
+ /** This contains environment.region */
+ private String zone;
+
+ private String tenant;
+
+ // This must contain applicationName.instanceName. TODO: Fix
+ private String app;
+
+ private int tco;
+ private float utilization;
+ private float waste;
+ Map<String, ClusterCost> cluster;
+
+ /** Create an empty (invalid) application cost */
+ public ApplicationCost() {}
+
+ public ApplicationCost(String zone, String tenant, String app, int tco, float utilization, float waste,
+ Map<String, ClusterCost> clusterCost) {
+ this.zone = zone;
+ this.tenant = tenant;
+ this.app = app;
+ this.tco = tco;
+ this.utilization = utilization;
+ this.waste = waste;
+ cluster = new HashMap<>(clusterCost);
+ }
+
+ public String getZone() {
+ return zone;
+ }
+
+ public void setZone(String zone) {
+ this.zone = zone;
+ }
+
+ public String getApp() {
+ return app;
+ }
+
+ public void setApp(String app) {
+ this.app = app;
+ }
+
+ public Map<String, ClusterCost> getCluster() {
+ return cluster;
+ }
+
+ public void setCluster(Map<String, ClusterCost> cluster) {
+ this.cluster = cluster;
+ }
+
+ public int getTco() {
+ return tco;
+ }
+
+ public void setTco(int tco) {
+ if (tco < 0) throw new IllegalArgumentException("TCO cannot be negative");
+ this.tco = tco;
+ }
+
+ public String getTenant() {
+ return tenant;
+ }
+
+ public void setTenant(String tenant) {
+ this.tenant = tenant;
+ }
+
+ public float getUtilization() {
+ return utilization;
+ }
+
+ public void setUtilization(float utilization) {
+ if (utilization < 0) throw new IllegalArgumentException("Utilization cannot be negative");
+ this.utilization = utilization;
+ }
+
+ public float getWaste() {
+ return waste;
+ }
+
+ public void setWaste(float waste) {
+ this.waste = waste;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java
new file mode 100644
index 00000000000..d9edf22d42c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Backend.java
@@ -0,0 +1,21 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.cost;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException;
+
+import java.util.List;
+
+/**
+ * Interface for retrieving cost data directly or indirectly from yamas and
+ * the noderepository.
+ *
+ *
+ * @author smorgrav
+ */
+public interface Backend {
+ List<ApplicationCost> getApplicationCost();
+ ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId appId) throws NotFoundCheckedException;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java
new file mode 100644
index 00000000000..1e41325a4fd
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/ClusterCost.java
@@ -0,0 +1,182 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.cost;
+
+import java.util.List;
+
+/**
+ * Cost data model for a cluster. I.e one cluster within one vespa application in one zone.
+ *
+ * @author smorgrav
+ */
+// TODO: Use doubles
+// TODO: Make immutable
+// TODO: Enforce constraints
+// TODO: Document content
+public class ClusterCost {
+
+ private int count;
+ private String resource;
+ private float utilization;
+ private int tco;
+ private String flavor;
+ private int waste;
+ private String type;
+ private float utilMem;
+ private float utilCpu;
+ private float utilDisk;
+ private float utilDiskBusy;
+ private float usageMem;
+ private float usageCpu;
+ private float usageDisk;
+ private float usageDiskBusy;
+ private List<String> hostnames;
+
+ /** Create an empty (invalid) cluster cost */
+ public ClusterCost() {}
+
+ public int getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ public String getFlavor() {
+ return flavor;
+ }
+
+ public void setFlavor(String flavor) {
+ this.flavor = flavor;
+ }
+
+ public List<String> getHostnames() {
+ return hostnames;
+ }
+
+ public void setHostnames(List<String> hostnames) {
+ this.hostnames = hostnames;
+ }
+
+ public String getResource() {
+ return resource;
+ }
+
+ public void setResource(String resource) {
+ this.resource = resource;
+ }
+
+ public int getTco() {
+ return tco;
+ }
+
+ public void setTco(int tco) {
+ this.tco = tco;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public float getUtilization() {
+ return utilization;
+ }
+
+ public void setUtilization(float utilization) {
+ validateUtilRatio(utilization);
+ this.utilization = utilization;
+ }
+
+ public int getWaste() {
+ return waste;
+ }
+
+ public void setWaste(int waste) {
+ this.waste = waste;
+ }
+
+ public float getUsageCpu() {
+ return usageCpu;
+ }
+
+ public void setUsageCpu(float usageCpu) {
+ validateUsageRatio(usageCpu);
+ this.usageCpu = usageCpu;
+ }
+
+ public float getUsageDisk() {
+ return usageDisk;
+ }
+
+ public void setUsageDisk(float usageDisk) {
+ validateUsageRatio(usageDisk);
+ this.usageDisk = usageDisk;
+ }
+
+ public float getUsageMem() {
+ return usageMem;
+ }
+
+ public void setUsageMem(float usageMem) {
+ validateUsageRatio(usageMem);
+ this.usageMem = usageMem;
+ }
+
+ public float getUtilCpu() {
+ return utilCpu;
+ }
+
+ public void setUtilCpu(float utilCpu) {
+ validateUtilRatio(utilCpu);
+ this.utilCpu = utilCpu;
+ }
+
+ public float getUtilDisk() {
+ return utilDisk;
+ }
+
+ public void setUtilDisk(float utilDisk) {
+ validateUtilRatio(utilDisk);
+ this.utilDisk = utilDisk;
+ }
+
+ public float getUtilMem() {
+ return utilMem;
+ }
+
+ public void setUtilMem(float utilMem) {
+ validateUsageRatio(utilMem);
+ this.utilMem = utilMem;
+ }
+
+ public float getUsageDiskBusy() {
+ return usageDiskBusy;
+ }
+
+ public void setUsageDiskBusy(float usageDiskBusy) {
+ validateUsageRatio(usageDiskBusy);
+ this.usageDiskBusy = usageDiskBusy;
+ }
+
+ public float getUtilDiskBusy() {
+ return utilDiskBusy;
+ }
+
+ public void setUtilDiskBusy(float utilDiskBusy) {
+ validateUtilRatio(utilDiskBusy);
+ this.utilDiskBusy = utilDiskBusy;
+ }
+
+ private void validateUsageRatio(float ratio) {
+ if (ratio < 0) throw new IllegalArgumentException("Usage cannot be negative");
+ if (ratio > 1) throw new IllegalArgumentException("Usage exceed 1 (using more than it has available)");
+ }
+
+ private void validateUtilRatio(float ratio) {
+ if (ratio < 0) throw new IllegalArgumentException("Utilization cannot be negative");
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java
new file mode 100644
index 00000000000..7297b60de5c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/Cost.java
@@ -0,0 +1,53 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.cost;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException;
+
+import java.util.List;
+
+/**
+ * Cost domain model declaration
+ *
+ * @author smorgrav
+ */
+public interface Cost {
+
+ /**
+ * Calculate a list of the applications that is wasting most
+ * in absolute terms. To improve utilization, it should make
+ * sense to focus on this list.
+ *
+ * @return An ordered set of applications with the highest potential for
+ * improved CPU utilization across all environments and regions.
+ */
+ List<ApplicationCost> getCPUAnalysis(int nofApplications);
+
+ /**
+ * Collect all information and format it as a Cvs blob for download.
+ *
+ * @return A String with comma separated values. Can be big!
+ */
+ String getCsvForLocalAnalysis();
+
+ /**
+ * Get application costs for all applications across all regions and environments
+ *
+ * @return A list of applications in given zone
+ */
+ List<ApplicationCost> getApplicationCost();
+
+ /**
+ * Get application costs for a given application instance in a given zone.
+ *
+ * @param env Environment like test, dev, perf, staging or prod
+ * @param region Region name like us-east-1
+ * @param app ApplicationId like tenant:application:instance
+ *
+ * @return A list of applications in given zone
+ */
+ ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId app)
+ throws NotFoundCheckedException;
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java
new file mode 100644
index 00000000000..088b1fa12bc
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/CostJsonModelAdapter.java
@@ -0,0 +1,93 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.cost;
+
+import com.yahoo.slime.Cursor;
+import com.yahoo.vespa.hosted.controller.api.cost.CostJsonModel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Converting from cost data model to the JSON data model used in the cost REST API.
+ *
+ * @author smorgrav
+ */
+public class CostJsonModelAdapter {
+
+ public static CostJsonModel.Application toJsonModel(ApplicationCost appCost) {
+ CostJsonModel.Application app = new CostJsonModel.Application();
+ app.zone = appCost.getZone();
+ app.tenant = appCost.getTenant();
+ app.app = appCost.getApp();
+ app.tco = appCost.getTco();
+ app.utilization = appCost.getUtilization();
+ app.waste = appCost.getWaste();
+ app.cluster = new HashMap<>();
+ Map<String, ClusterCost> clusterMap = appCost.getCluster();
+ for (String key : clusterMap.keySet()) {
+ app.cluster.put(key, toJsonModel(clusterMap.get(key)));
+ }
+
+ return app;
+ }
+
+ public static void toSlime(ApplicationCost appCost, Cursor object) {
+ object.setString("zone", appCost.getZone());
+ object.setString("tenant", appCost.getTenant());
+ object.setString("app", appCost.getApp());
+ object.setLong("tco", appCost.getTco());
+ object.setDouble("utilization", appCost.getUtilization());
+ object.setDouble("waste", appCost.getWaste());
+ Cursor clustersObject = object.setObject("cluster");
+ for (Map.Entry<String, ClusterCost> clusterEntry : appCost.getCluster().entrySet())
+ toSlime(clusterEntry.getValue(), clustersObject.setObject(clusterEntry.getKey()));
+ }
+
+ public static CostJsonModel.Cluster toJsonModel(ClusterCost clusterCost) {
+ CostJsonModel.Cluster cluster = new CostJsonModel.Cluster();
+ cluster.count = clusterCost.getCount();
+ cluster.resource = clusterCost.getResource();
+ cluster.utilization = clusterCost.getUtilization();
+ cluster.tco = clusterCost.getTco();
+ cluster.flavor = clusterCost.getFlavor();
+ cluster.waste = clusterCost.getWaste();
+ cluster.type = clusterCost.getType();
+ cluster.util = new CostJsonModel.HardwareResources();
+ cluster.util.cpu = clusterCost.getUtilCpu();
+ cluster.util.mem = clusterCost.getUtilMem();
+ cluster.util.disk = clusterCost.getUtilDisk();
+ cluster.usage = new CostJsonModel.HardwareResources();
+ cluster.usage.cpu = clusterCost.getUsageCpu();
+ cluster.usage.mem = clusterCost.getUsageMem();
+ cluster.usage.disk = clusterCost.getUsageDisk();
+ cluster.hostnames = new ArrayList<>(clusterCost.getHostnames());
+ cluster.usage.diskBusy = clusterCost.getUsageDiskBusy();
+ cluster.util.diskBusy = clusterCost.getUtilDiskBusy();
+ return cluster;
+ }
+
+ private static void toSlime(ClusterCost clusterCost, Cursor object) {
+ object.setLong("count", clusterCost.getCount());
+ object.setString("resource", clusterCost.getResource());
+ object.setDouble("utilization", clusterCost.getUtilization());
+ object.setLong("tco", clusterCost.getTco());
+ object.setString("flavor", clusterCost.getFlavor());
+ object.setLong("waste", clusterCost.getWaste());
+ object.setString("type", clusterCost.getType());
+ Cursor utilObject = object.setObject("util");
+ utilObject.setDouble("cpu", clusterCost.getUtilCpu());
+ utilObject.setDouble("mem", clusterCost.getUtilMem());
+ utilObject.setDouble("disk", clusterCost.getUtilDisk());
+ utilObject.setDouble("diskBusy", clusterCost.getUtilDiskBusy());
+ Cursor usageObject = object.setObject("usage");
+ usageObject.setDouble("cpu", clusterCost.getUsageCpu());
+ usageObject.setDouble("mem", clusterCost.getUsageMem());
+ usageObject.setDouble("disk", clusterCost.getUsageDisk());
+ usageObject.setDouble("diskBusy", clusterCost.getUsageDiskBusy());
+ Cursor hostnamesArray = object.setArray("hostnames");
+ for (String hostname : clusterCost.getHostnames())
+ hostnamesArray.addString(hostname);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java
new file mode 100644
index 00000000000..f08e6cc9b36
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/cost/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.cost;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
new file mode 100644
index 00000000000..f70afb3a0a0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.dns;
+
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * An in-memory name service for testing purposes.
+ *
+ * @author mpolden
+ */
+public class MemoryNameService implements NameService {
+
+ private final Set<Record> records = new HashSet<>();
+
+ @Override
+ public RecordId createCname(String alias, String canonicalName) {
+ records.add(new Record("CNAME", alias, canonicalName));
+ return new RecordId(UUID.randomUUID().toString());
+ }
+
+ @Override
+ public Optional<Record> findRecord(Record.Type type, String name) {
+ return records.stream()
+ .filter(record -> record.type() == type && record.name().equals(name))
+ .findFirst();
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java
new file mode 100644
index 00000000000..2ccce23b60c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/NameService.java
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.dns;
+
+import java.util.Optional;
+
+/**
+ * A managed DNS service.
+ *
+ * @author mpolden
+ */
+public interface NameService {
+
+ /**
+ * Create a new CNAME record
+ *
+ * @param alias The alias to create
+ * @param canonicalName The canonical name which the alias should point to. This must be a domain.
+ */
+ RecordId createCname(String alias, String canonicalName);
+
+ /** Find record by type and name */
+ Optional<Record> findRecord(Record.Type type, String name);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java
new file mode 100644
index 00000000000..0782a82da79
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java
@@ -0,0 +1,74 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.dns;
+
+import java.util.Objects;
+
+/**
+ * A basic representation of a DNS resource record, containing only the record type, name and value.
+ *
+ * @author mpolden
+ */
+public class Record {
+
+ private final Type type;
+ private final String name;
+ private final String value;
+
+ public Record(Type type, String name, String value) {
+ this.type = type;
+ this.name = name;
+ this.value = value;
+ }
+
+ public Record(String type, String name, String value) {
+ this(Type.valueOf(type), name, value);
+ }
+
+ public Type type() {
+ return type;
+ }
+
+ public String value() {
+ return value;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public enum Type {
+ A,
+ AAAA,
+ CNAME,
+ MX,
+ NS,
+ PTR,
+ SOA,
+ SRV,
+ TXT
+ }
+
+ @Override
+ public String toString() {
+ return "Record{" +
+ "type=" + type +
+ ", name='" + name + '\'' +
+ ", value='" + value + '\'' +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof Record)) return false;
+ Record record = (Record) o;
+ return type == record.type &&
+ Objects.equals(name, record.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, name);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java
new file mode 100644
index 00000000000..9c47be12855
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/RecordId.java
@@ -0,0 +1,27 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.dns;
+
+/**
+ * Unique identifier for a resource record.
+ *
+ * @author mpolden
+ */
+public class RecordId {
+
+ private final String id;
+
+ public RecordId(String id) {
+ this.id = id;
+ }
+
+ public String id() {
+ return id;
+ }
+
+ @Override
+ public String toString() {
+ return "RecordId{" +
+ "id='" + id + '\'' +
+ '}';
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java
new file mode 100644
index 00000000000..e075b544ce8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.dns;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java
new file mode 100644
index 00000000000..fc242a360f6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/EntityService.java
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.entity;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A service which provides access to business-specific entities.
+ *
+ * @author mpolden
+ */
+public interface EntityService {
+
+ /** List all properties known by the service */
+ Map<PropertyId, Property> listProperties();
+
+ /** List all groups of which user is a member */
+ Set<UserGroup> getUserGroups(UserId user);
+
+ /** Whether user is a member of the group */
+ boolean isGroupMember(UserId user, UserGroup group);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java
new file mode 100644
index 00000000000..e5c2bbedae4
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/MemoryEntityService.java
@@ -0,0 +1,37 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.entity;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author mpolden
+ */
+public class MemoryEntityService implements EntityService {
+
+ @Override
+ public Map<PropertyId, Property> listProperties() {
+ Map<PropertyId, Property> properties = new HashMap<>();
+ properties.put(new PropertyId("1234"), new Property("foo"));
+ properties.put(new PropertyId("4321"), new Property("bar"));
+ return Collections.unmodifiableMap(properties);
+ }
+
+ @Override
+ public Set<UserGroup> getUserGroups(UserId userId) {
+ return Collections.singleton(new UserGroup("vespa"));
+ }
+
+ @Override
+ public boolean isGroupMember(UserId userId, UserGroup userGroup) {
+ return true;
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java
new file mode 100644
index 00000000000..1e74f4ca372
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/entity/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.entity;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java
new file mode 100644
index 00000000000..1cb3f73441b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHub.java
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.github;
+
+/**
+ * @author mpolden
+ */
+public interface GitHub {
+
+ GitSha getCommit(String owner, String repo, String ref);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java
new file mode 100644
index 00000000000..9a398ef7cb5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitHubMock.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.github;
+
+import java.time.Instant;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * @author jvenstad
+ */
+public class GitHubMock implements GitHub {
+
+ private final Map<String, GitSha> tags = new HashMap<>();
+ private boolean mockAny = true;
+
+ @Override
+ public GitSha getCommit(String owner, String repo, String ref) {
+ if (mockAny) {
+ String sha = UUID.randomUUID().toString();
+ return new GitSha(sha, new GitSha.GitCommit(new GitSha.GitAuthor("foo", "foo@foo.tld",
+ Date.from(Instant.EPOCH))));
+ }
+ if (tags.containsKey(ref)) {
+ return tags.get(ref);
+ }
+ throw new IllegalArgumentException("Unknown ref: " + ref);
+ }
+
+ public GitHubMock knownTag(String tag, String sha) {
+ this.tags.put(tag, new GitSha(sha, new GitSha.GitCommit(
+ new GitSha.GitAuthor("foo", "foo@foo.tld", Date.from(Instant.EPOCH)))));
+ return this;
+ }
+
+ public GitHubMock mockAny(boolean mockAny) {
+ this.mockAny = mockAny;
+ return this;
+ }
+
+ public GitHubMock reset() {
+ tags.clear();
+ mockAny = true;
+ return this;
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java
new file mode 100644
index 00000000000..4aac98f1708
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/GitSha.java
@@ -0,0 +1,56 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.github;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Date;
+
+/**
+ * @author mpolden
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class GitSha {
+
+ @JsonProperty("sha")
+ public final String sha;
+
+ @JsonProperty("commit")
+ public final GitCommit commit;
+
+ @JsonCreator
+ public GitSha(@JsonProperty("sha") String sha, @JsonProperty("commit") GitCommit commit) {
+ this.sha = sha;
+ this.commit = commit;
+ }
+
+ public static class GitCommit {
+ @JsonProperty("author")
+ public final GitAuthor author;
+
+ @JsonCreator
+ public GitCommit(@JsonProperty("author") GitAuthor author) {
+ this.author = author;
+ }
+ }
+
+ public static class GitAuthor {
+
+ @JsonProperty("name")
+ public final String name;
+ @JsonProperty("email")
+ public final String email;
+ @JsonProperty("date")
+ public final Date date;
+
+ @JsonCreator
+ public GitAuthor(@JsonProperty("name") String name, @JsonProperty("email") String email,
+ @JsonProperty("date") Date date) {
+ this.name = name;
+ this.email = email;
+ this.date = date;
+ }
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java
new file mode 100644
index 00000000000..ec20c05c374
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/github/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.github;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java
new file mode 100644
index 00000000000..30bf23f18a9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/Jira.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+public interface Jira {
+
+ List<JiraIssue> searchByProjectAndSummary(String project, String summary);
+
+ JiraIssue createIssue(JiraCreateIssue issue);
+
+ void commentIssue(JiraIssue issue, JiraComment comment);
+
+ void addAttachment(JiraIssue issue, String filename, String fileContent);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java
new file mode 100644
index 00000000000..2d67b720fe0
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraComment.java
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class JiraComment {
+
+ public final String body;
+
+ @JsonCreator
+ public JiraComment(@JsonProperty("body") String body) {
+ this.body = body;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java
new file mode 100644
index 00000000000..e5e35af4475
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraCreateIssue.java
@@ -0,0 +1,86 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class JiraCreateIssue {
+
+ @JsonProperty("fields")
+ public final JiraFields fields;
+
+ public JiraCreateIssue(JiraFields fields) {
+ this.fields = fields;
+ }
+
+ public static class JiraFields {
+ @JsonProperty("summary")
+ public final String summary;
+
+ @JsonProperty("description")
+ public final String description;
+
+ @JsonProperty("project")
+ public final JiraProject project;
+
+ @JsonProperty("issuetype")
+ public final JiraIssueType issueType;
+
+ @JsonProperty("components")
+ public final List<JiraComponent> components;
+
+ public JiraFields(
+ JiraProject project,
+ String summary,
+ String description,
+ JiraIssueType issueType,
+ List<JiraComponent> components) {
+ this.project = project;
+ this.summary = summary;
+ this.description = description;
+ this.issueType = issueType;
+ this.components = components;
+ }
+
+
+ public static class JiraProject {
+ public static final JiraProject VESPA = new JiraProject("VESPA");
+
+ @JsonProperty("key")
+ public final String key;
+
+ public JiraProject(String key) {
+ this.key = key;
+ }
+ }
+
+ public static class JiraIssueType {
+ public static final JiraIssueType DEFECT = new JiraIssueType("Defect");
+
+ @JsonProperty("name")
+ public final String name;
+
+ public JiraIssueType(String name) {
+ this.name = name;
+ }
+ }
+
+ public static class JiraComponent {
+ public static final JiraComponent COREDUMPS = new JiraComponent("CoreDumps");
+
+ @JsonProperty("name")
+ public final String name;
+
+
+ public JiraComponent(String name) {
+ this.name = name;
+ }
+ }
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java
new file mode 100644
index 00000000000..d88e75d3a58
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssue.java
@@ -0,0 +1,45 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.Instant;
+import java.util.Date;
+
+/**
+ * @author mpolden
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class JiraIssue {
+ public final String key;
+ private final Fields fields;
+
+ @JsonCreator
+ public JiraIssue(@JsonProperty("key") String key, @JsonProperty("fields") Fields fields) {
+ this.key = key;
+ this.fields = fields;
+ }
+
+ public Instant lastUpdated() {
+ return fields.lastUpdated;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class Fields {
+ final Instant lastUpdated;
+
+ @JsonCreator
+ public Fields(
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'hh:mm:ss.SSSZ", timezone = "UTC")
+ @JsonProperty("updated") Date updated) {
+ lastUpdated = updated.toInstant();
+ }
+
+ public Fields(Instant instant) {
+ this.lastUpdated = instant;
+ }
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java
new file mode 100644
index 00000000000..809ac8360bb
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraIssues.java
@@ -0,0 +1,22 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class JiraIssues {
+ public final List<JiraIssue> issues;
+
+ @JsonCreator
+ public JiraIssues(@JsonProperty("issues") List<JiraIssue> issues) {
+ this.issues = issues == null ? Collections.emptyList() : issues;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java
new file mode 100644
index 00000000000..da653ddd8a8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/JiraMock.java
@@ -0,0 +1,50 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author jvenstad
+ */
+// TODO: Make mock.
+public class JiraMock implements Jira {
+
+ public final Map<String, JiraCreateIssue.JiraFields> issues = new HashMap<>();
+
+ private Long counter = 0L;
+
+ @Override
+ public List<JiraIssue> searchByProjectAndSummary(String project, String summary) {
+ return issues.entrySet().stream()
+ .filter(entry -> entry.getValue().project.key.equals(project))
+ .filter(entry -> entry.getValue().summary.contains(summary))
+ .map(entry -> new JiraIssue(entry.getKey(), new JiraIssue.Fields(Instant.now())))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public JiraIssue createIssue(JiraCreateIssue issueData) {
+ JiraIssue issue = uniqueKey();
+ issues.put(issue.key, issueData.fields);
+ return issue;
+ }
+
+ @Override
+ public void commentIssue(JiraIssue issue, JiraComment comment) {
+ // Add mock when relevant.
+ }
+
+ @Override
+ public void addAttachment(JiraIssue issue, String filename, String fileContent) {
+ // Add mock when relevant.
+ }
+
+ private JiraIssue uniqueKey() {
+ return new JiraIssue((++counter).toString(), new JiraIssue.Fields(Instant.now()));
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java
new file mode 100644
index 00000000000..efe356c69e9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/jira/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.jira;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java
new file mode 100644
index 00000000000..265d57cadd8
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java
new file mode 100644
index 00000000000..d49d6a9e4c2
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java
@@ -0,0 +1,16 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.routing;
+
+import java.util.Map;
+
+/**
+ * A global routing service.
+ *
+ * @author mpolden
+ */
+public interface GlobalRoutingService {
+
+ /** Returns the health status for each endpoint behind the given rotation name */
+ Map<String, RotationStatus> getHealthStatus(String rotationName);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java
new file mode 100644
index 00000000000..9f1ac1b1f0b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.routing;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author bratseth
+ */
+public class MemoryGlobalRoutingService implements GlobalRoutingService {
+
+ @Override
+ public Map<String, RotationStatus> getHealthStatus(String rotationName) {
+ HashMap<String, RotationStatus> map = new HashMap<>();
+ map.put("prod.us-west-1", RotationStatus.IN);
+ return Collections.unmodifiableMap(map);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java
new file mode 100644
index 00000000000..8c59bb44fa1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RotationStatus.java
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.routing;
+
+/**
+ * Represents the health status of a global rotation.
+ *
+ * @author andreer
+ */
+public enum RotationStatus {
+ IN, OUT, UNKNOWN
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java
new file mode 100644
index 00000000000..a4bf733bd2c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java
@@ -0,0 +1,30 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.routing;
+
+/**
+ * @author smorgrav
+ */
+public class RoutingEndpoint {
+
+ private final boolean isGlobal;
+ private final String endpoint;
+
+ public RoutingEndpoint(String endpoint, boolean isGlobal) {
+ this.endpoint = endpoint;
+ this.isGlobal = isGlobal;
+ }
+
+ /**
+ * @return True if the endpoint is global
+ */
+ public boolean isGlobal() {
+ return isGlobal;
+ }
+
+ /*
+ * @return The URI for the endpoint
+ */
+ public String getEndpoint() {
+ return endpoint;
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java
new file mode 100644
index 00000000000..276e19da8f6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java
@@ -0,0 +1,19 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.routing;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+
+import java.util.List;
+
+/**
+ * @author bratseth
+ * @author smorgrav
+ */
+public interface RoutingGenerator {
+
+ /**
+ * @param deploymentId Specifying an application in a zone
+ * @return List of endpoints for that deploymentId
+ */
+ List<RoutingEndpoint> endpoints(DeploymentId deploymentId);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java
new file mode 100644
index 00000000000..25374003ec1
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.routing;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java
new file mode 100644
index 00000000000..98c664eb07d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/KeyService.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.security;
+
+/**
+ * A service for retrieving secrets, such as API keys, private keys and passwords.
+ *
+ * @author mpolden
+ */
+public interface KeyService {
+
+ String getSecret(String key);
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java
new file mode 100644
index 00000000000..296eebf8ea5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/security/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.security;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java
new file mode 100644
index 00000000000..9114cf20ccc
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/ContactsMock.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.stubs;
+
+import com.yahoo.vespa.hosted.controller.api.integration.Contacts;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author mpolden
+ */
+public class ContactsMock implements Contacts {
+
+ private final Map<Long, List<UserContact>> userContacts = new HashMap<>();
+
+ public void addContact(long propertyId, List<UserContact> contacts) {
+ userContacts.put(propertyId, contacts);
+ }
+
+ public List<UserContact> userContactsFor(long propertyId) {
+ return userContacts.get(propertyId);
+ }
+
+ @Override
+ public URI contactsUri(long propertyId) {
+ return URI.create("http://contacts.test?propertyId=" + propertyId);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java
new file mode 100644
index 00000000000..160f80076bd
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingIssues.java
@@ -0,0 +1,87 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.stubs;
+
+import com.yahoo.vespa.hosted.controller.api.integration.Issues;
+
+import java.time.Instant;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+/**
+ * An memory backed implementation of the Issues API which logs changes and does nothing else.
+ *
+ * @author bratseth
+ */
+public class LoggingIssues implements Issues {
+
+ private static final Logger log = Logger.getLogger(LoggingIssues.class.getName());
+
+ /** Used to fabricate unique issue ids */
+ private AtomicLong issueIdSequence = new AtomicLong(0);
+
+ // These two maps should have precisely the same keys
+ private final Map<String, Issue> issues = new HashMap<>();
+ private final Map<String, IssueInfo> issueInfos = new HashMap<>();
+
+ @Override
+ public IssueInfo fetch(String issueId) {
+ return issueInfos.getOrDefault(issueId,
+ new IssueInfo(issueId, null, Instant.ofEpochMilli(0), null, IssueInfo.Status.noCategory));
+ }
+
+ @Override
+ public List<IssueInfo> fetchSimilarTo(Issue issue) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public String file(Issue issue) {
+ log.info("Want to file " + issue);
+ String issueId = "issue-" + issueIdSequence.getAndIncrement();
+ issues.put(issueId, issue);
+ issueInfos.put(issueId, new IssueInfo(issueId, null, Instant.now(), null, IssueInfo.Status.noCategory));
+ return issueId;
+ }
+
+ @Override
+ public void update(String issueId, String description) {
+ log.info("Want to update " + issueId);
+ issues.put(issueId, requireIssue(issueId).withDescription(description));
+ }
+
+ @Override
+ public void reassign(String issueId, String assignee) {
+ log.info("Want to reassign issue " + issueId + " to " + assignee);
+ issueInfos.put(issueId, requireInfo(issueId).withAssignee(Optional.of(assignee)));
+ }
+
+ @Override
+ public void addWatcher(String issueId, String watcher) {
+ log.info("Want to add watcher " + watcher + " to issue " + issueId);
+ }
+
+ @Override
+ public void comment(String issueId, String comment) {
+ log.info("Want to comment on issue " + issueId);
+ }
+
+ private Issue requireIssue(String issueId) {
+ Issue issue = issues.get(issueId);
+ if (issue == null)
+ throw new IllegalArgumentException("No issue with id '" + issueId + "'");
+ return issue;
+ }
+
+ private IssueInfo requireInfo(String issueId) {
+ IssueInfo info = issueInfos.get(issueId);
+ if (info == null)
+ throw new IllegalArgumentException("No issue info with id '" + issueId + "'");
+ return info;
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java
new file mode 100644
index 00000000000..53a31933e03
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/PropertiesMock.java
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.stubs;
+
+import com.yahoo.vespa.hosted.controller.api.integration.Issues;
+import com.yahoo.vespa.hosted.controller.api.integration.Properties;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author mpolden
+ */
+public class PropertiesMock implements Properties {
+
+ private final Map<Long, Issues.Classification> projects = new HashMap<>();
+
+ public void addClassification(long propertyId, String classification) {
+ projects.put(propertyId, new Issues.Classification(classification));
+ }
+
+ public Optional<Issues.Classification> classificationFor(long propertyId) {
+ return Optional.ofNullable(projects.get(propertyId));
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java
new file mode 100644
index 00000000000..2aab38dc66d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/package-info.java
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * No-dependency implementations of integration interfaces for setups where we want to avoid contacting
+ * certain thirds-party systems.
+ *
+ * @author bratseth
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.stubs;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
new file mode 100644
index 00000000000..e7bdf786c8c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.zone;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Provides information about zones in a hosted Vespa system.
+ *
+ * @author mpolden
+ */
+public interface ZoneRegistry {
+
+ SystemName system();
+ List<Zone> zones();
+ Optional<Zone> getZone(Environment environment, RegionName region);
+ List<URI> getConfigServerUris(Environment environment, RegionName region);
+ Optional<URI> getLogServerUri(Environment environment, RegionName region);
+ Optional<Duration> getDeploymentTimeToLive(Environment environment, RegionName region);
+ URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application);
+ URI getDashboardUri();
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java
new file mode 100644
index 00000000000..148564a373f
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.integration.zone;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java
new file mode 100644
index 00000000000..78a6750aedb
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/nonpublic/HeaderFields.java
@@ -0,0 +1,14 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.nonpublic;
+
+/**
+ * Non public header fields that are not part of the public api.
+ *
+ * Placed here since this is the only module we own that both the
+ * command-line client and controller-server depend on.
+ *
+ * @author Tony Vaagenes
+ */
+public class HeaderFields {
+ public static final String USER_ID_HEADER_FIELD = "vespa.hosted.trusted.username";
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java
new file mode 100644
index 00000000000..ed3e69bcac7
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/Rotation.java
@@ -0,0 +1,39 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.rotation;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId;
+
+import java.util.Objects;
+
+/**
+ * Represents a global routing rotation.
+ *
+ * @author Oyvind Gronnesby
+ */
+public class Rotation {
+
+ /** The ID of the allocated rotation. This value is generated by global routing system. */
+ public final RotationId rotationId;
+
+ /** The global name which the allocated rotation points to */
+ public final String rotationName;
+
+ public Rotation(RotationId rotationId, String rotationName) {
+ this.rotationId = Objects.requireNonNull(rotationId);
+ this.rotationName = Objects.requireNonNull(rotationName);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof Rotation)) return false;
+ final Rotation rotation = (Rotation) o;
+ return rotationId.equals(rotation.rotationId) && rotationName.equals(rotation.rotationName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rotationId, rotationName);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java
new file mode 100644
index 00000000000..1626158a489
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/rotation/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.rotation;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java
new file mode 100644
index 00000000000..65c5e0f9365
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/StatusPageResource.java
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.statuspage;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * @author andreer
+ */
+@Path("/v1/")
+@Produces(MediaType.APPLICATION_JSON)
+public interface StatusPageResource {
+
+ @GET
+ @Path("{page}")
+ @Produces(MediaType.APPLICATION_JSON)
+ JsonNode statusPage(@PathParam("page") String page, @QueryParam("since") String since);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java
new file mode 100644
index 00000000000..3f9117bf931
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/statuspage/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.statuspage;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java
new file mode 100644
index 00000000000..7bb4bfc6467
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneApi.java
@@ -0,0 +1,35 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.zone.v1;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.util.List;
+
+/**
+ * Used by build system and command-line tool.
+ *
+ * @author smorgrav
+ */
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+@Path(ZoneApi.API_VERSION)
+public interface ZoneApi {
+
+ String API_VERSION = "v1";
+
+ @GET
+ @Path("")
+ List<ZoneReference.Environment> listEnvironments();
+
+ @GET
+ @Path("/environment/{environment}")
+ List<ZoneReference.Region> listRegions(@PathParam("environment") String env);
+
+ @GET
+ @Path("/environment/{environment}/default")
+ ZoneReference.Region defaultRegion(@PathParam("environment") String env);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java
new file mode 100644
index 00000000000..82d03d72acd
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/ZoneReference.java
@@ -0,0 +1,64 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.zone.v1;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.net.URI;
+
+/**
+ * @author smorgrav
+ */
+public class ZoneReference {
+
+ public static class Environment {
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("url")
+ private URI url;
+
+ public String getName() {
+ return name;
+ }
+
+ public Environment setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public URI getUrl() {
+ return url;
+ }
+
+ public Environment setUrl(URI url) {
+ this.url = url;
+ return this;
+ }
+ }
+
+ public static class Region {
+ @JsonProperty("name")
+ private String name;
+
+ @JsonProperty("url")
+ private URI url;
+
+ public String getName() {
+ return name;
+ }
+
+ public Region setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public URI getUrl() {
+ return url;
+ }
+
+ public Region setUrl(URI url) {
+ this.url = url;
+ return this;
+ }
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java
new file mode 100644
index 00000000000..e3275ff35fa
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v1/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.zone.v1;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java
new file mode 100644
index 00000000000..97d99e262b5
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneApiV2.java
@@ -0,0 +1,110 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.zone.v2;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+/**
+ * Aka the controller proxy service.
+ *
+ * Proxies calls to correct config server with the additional feature of
+ * retry and fail detection (ping).
+ */
+@Path(ZoneApiV2.API_VERSION)
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public interface ZoneApiV2 {
+
+ String API_VERSION = "v2";
+
+ @GET
+ @Path("/")
+ ZoneReferences listZones();
+
+ @GET
+ @Path("/{environment}/{region}/{proxy_request: .+}")
+ Response proxyGet(
+ @PathParam("environment") String env,
+ @PathParam("region") String region,
+ @PathParam("proxy_request") String proxyRequest,
+ @Context HttpServletRequest request);
+
+ @POST
+ @Path("/{environment}/{region}/{proxy_request: .+}")
+ Response proxyPost(
+ @PathParam("environment") String env,
+ @PathParam("region") String region,
+ @PathParam("proxy_request") String proxyRequest,
+ @Context HttpServletRequest request);
+
+ @PUT
+ @Path("/{environment}/{region}/{proxy_request: .+}")
+ Response proxyPut(
+ @PathParam("environment") String env,
+ @PathParam("region") String region,
+ @PathParam("proxy_request") String proxyRequest,
+ @Context HttpServletRequest request);
+
+ @DELETE
+ @Path("/{environment}/{region}/{proxy_request: .+}")
+ Response proxyDelete(
+ @PathParam("environment") String env,
+ @PathParam("region") String region,
+ @PathParam("proxy_request") String proxyRequest,
+ @Context HttpServletRequest request);
+
+ // Explicit mappings of some proxy requests (to enable creation of proxy clients with javax.ws.rs)
+
+ @GET
+ @Path("/{environmentId}/{regionId}/application/v2/tenant/{tenantId}/application/{applicationId}/environment/{environmentId}/region/{regionId}/instance/{instanceId}/serviceconverge")
+ Response waitForConfigConvergeV2(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId,
+ @QueryParam("timeout") long timeoutInSeconds);
+ @GET
+ @Path("/{environmentId}/{regionId}/application/v2/tenant/{tenantId}/application/{applicationId}/environment/{environmentId}/region/{regionId}/instance/{instanceId}/serviceconverge/{host}")
+ Response waitForConfigConvergeV2(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("instanceId") InstanceId instanceId,
+ @PathParam("host") String host,
+ @QueryParam("timeout") long timeoutInSeconds);
+
+ @GET
+ @Path("/{environmentId}/{regionId}/config/v2/tenant/{tenantId}/application/{applicationId}/prelude.fastsearch.documentdb-info/{clusterid}/search/cluster.{clusterid}")
+ JsonNode getConfigWithDocumentTypes(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @PathParam("clusterid") String clusterid,
+ @QueryParam("timeout") long timeoutInSeconds);
+
+ @GET
+ @Path("/{environmentId}/{regionId}/config/v2/tenant/{tenantId}/application/{applicationId}/cloud.config.cluster-list")
+ JsonNode getVespaConfigClusterList(@PathParam("tenantId") TenantId tenantId,
+ @PathParam("applicationId") ApplicationId applicationId,
+ @PathParam("environmentId") EnvironmentId environmentId,
+ @PathParam("regionId") RegionId regionId,
+ @QueryParam("timeout") long timeoutInSeconds);
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java
new file mode 100644
index 00000000000..95dc1c2ee7c
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReference.java
@@ -0,0 +1,27 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.zone.v2;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.hosted.controller.api.configserver.Environment;
+import com.yahoo.vespa.hosted.controller.api.configserver.Region;
+
+/**
+ * @author mpolden
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ZoneReference {
+
+ @JsonProperty("environment")
+ public final Environment environment;
+ @JsonProperty("region")
+ public final Region region;
+
+ @JsonCreator
+ public ZoneReference(@JsonProperty("environment") Environment environment, @JsonProperty("region") Region region) {
+ this.environment = environment;
+ this.region = region;
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java
new file mode 100644
index 00000000000..3a219afa0a6
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/ZoneReferences.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.zone.v2;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Wire format for listing the controller URIs for all the available zones
+ *
+ * @author smorgrav
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ZoneReferences {
+
+ @JsonProperty("uris")
+ public final List<String> uris;
+
+ @JsonProperty("zones")
+ public final List<ZoneReference> zones;
+
+ @JsonCreator
+ public ZoneReferences(@JsonProperty("uris") List<String> uris, @JsonProperty("zones") List<ZoneReference> zones) {
+ this.uris = Collections.unmodifiableList(uris);
+ this.zones = Collections.unmodifiableList(zones);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java
new file mode 100644
index 00000000000..5d4b1310981
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/zone/v2/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api.zone.v2;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java
new file mode 100644
index 00000000000..1cdff0f920b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/ContextAttributes.java
@@ -0,0 +1,13 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.common;
+
+/**
+ * Constants for request context attributes used in our APIs.
+ *
+ * @author mpolden
+ */
+public interface ContextAttributes {
+
+ String SECURITY_CONTEXT_ATTRIBUTE = "vespa.hosted.security_context";
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java
new file mode 100644
index 00000000000..a55a7e2bdfc
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/NotFoundCheckedException.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.common;
+
+/**
+ * We have tons of places where we throw exceptions when
+ * some hosted resource is not found. This is usually
+ * done with IllegalArgumentExceptions, java.ws.rs exceptions or
+ * the servermodel runtime exceptions in the controller-server module.
+ *
+ * This is a checked alternative to do the same thing.
+ *
+ * @author smorgrav
+ */
+public class NotFoundCheckedException extends Exception {
+
+ public NotFoundCheckedException() {
+ super();
+ }
+
+ public NotFoundCheckedException(String msg) {
+ super(msg);
+ }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java
new file mode 100644
index 00000000000..95decd86e8b
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/common/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.common;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java
new file mode 100644
index 00000000000..4a02fe23dec
--- /dev/null
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeployOptionsTest.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mortent
+ */
+public class DeployOptionsTest {
+
+ @Test
+ public void it_serializes_version() throws IOException {
+ DeployOptions options = new DeployOptions(Optional.empty(), Optional.of(new Version("6.98.227")), false, false);
+ final ObjectMapper objectMapper = new ObjectMapper()
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+ .registerModule(new Jdk8Module());
+
+ String string = objectMapper.writeValueAsString(options);
+ assertEquals("{\"screwdriverBuildJob\":null,\"vespaVersion\":\"6.98.227\",\"ignoreValidationErrors\":false,\"deployCurrentVersion\":false}", string);
+ }
+}
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java
new file mode 100644
index 00000000000..56825cf7c61
--- /dev/null
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java
@@ -0,0 +1,153 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.identifiers;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class IdentifierTest {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void existing_tenant_id_not_empty() {
+ new TenantId("");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void existing_tenant_id_must_check_pattern() {
+ new TenantId("`");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void default_not_allowed_for_tenants() {
+ new TenantId("default");
+ }
+
+ @Test
+ public void existing_tenant_id_must_accept_valid_id() {
+ new TenantId("msbe");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void existing_tenant_id_cannot_be_uppercase() {
+ new TenantId("MixedCaseTenant");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void existing_tenant_id_cannot_contain_dots() {
+ new TenantId("tenant.with.dots");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_tenant_id_cannot_contain_underscore() {
+ TenantId.validate("underscore_tenant");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_tenant_id_cannot_contain_dot() {
+ TenantId.validate("tenant.with.dots");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_tenant_id_cannot_contain_uppercase() {
+ TenantId.validate("UppercaseTenant");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_tenant_id_cannot_start_with_dash() {
+ TenantId.validate("-tenant");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_tenant_id_cannot_end_with_dash() {
+ TenantId.validate("tenant-");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void existing_application_id_cannot_be_uppercase() {
+ new ApplicationId("MixedCaseApplication");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void existing_application_id_cannot_contain_dots() {
+ new ApplicationId("application.with.dots");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_application_id_cannot_contain_underscore() {
+ ApplicationId.validate("underscore_application");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_application_id_cannot_contain_dot() {
+ ApplicationId.validate("application.with.dots");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_application_id_cannot_contain_uppercase() {
+ ApplicationId.validate("UppercaseApplication");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_application_id_cannot_start_with_dash() {
+ ApplicationId.validate("-application");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void new_application_id_cannot_end_with_dash() {
+ ApplicationId.validate("application-");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void instance_id_cannot_be_uppercase() {
+ new InstanceId("MixedCaseInstance");
+ }
+
+ @Test
+ public void rotation_id_may_contain_dot() {
+ new RotationId("rotation.id.with.dot");
+ }
+
+ @Test
+ public void user_tenant_id_does_not_contain_underscore() {
+ assertEquals("by-under-score-user", new UserId("under_score_user").toTenantId().id());
+ }
+
+ @Test
+ public void athens_parent_domain_is_without_name_suffix() {
+ assertEquals(new AthensDomain("yby.john"), new AthensDomain("yby.john.myapp").getParent());
+ }
+
+ @Test
+ public void athens_domain_name_is_last_suffix() {
+ assertEquals("myapp", new AthensDomain("yby.john.myapp").getName());
+ }
+
+ @Test
+ public void domain_without_dot_is_toplevel() {
+ assertTrue(new AthensDomain("toplevel").isTopLevelDomain());
+ assertFalse(new AthensDomain("not.toplevel").isTopLevelDomain());
+ }
+
+ @Test
+ public void dns_names_has_no_underscore() {
+ assertEquals("a-b-c", new ApplicationId("a_b_c").toDns());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void identifiers_cannot_be_named_api() {
+ new ApplicationId("api");
+ }
+
+
+ @Test
+ public void application_instance_id_dotted_string_is_subindentifers_concatinated_with_dots() {
+ DeploymentId id = new DeploymentId(com.yahoo.config.provision.ApplicationId.from("tenant", "application", "instance"),
+ new Zone(Environment.prod, RegionName.from("region")));
+ assertEquals("tenant.application.prod.region.instance", id.dottedString());
+ }
+}
diff --git a/controller-server/OWNERS b/controller-server/OWNERS
new file mode 100644
index 00000000000..e6a0537ba53
--- /dev/null
+++ b/controller-server/OWNERS
@@ -0,0 +1,2 @@
+bratseth
+mpolden
diff --git a/controller-server/pom.xml b/controller-server/pom.xml
new file mode 100644
index 00000000000..f9e84693452
--- /dev/null
+++ b/controller-server/pom.xml
@@ -0,0 +1,211 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ </parent>
+ <artifactId>controller-server</artifactId>
+ <packaging>container-plugin</packaging>
+ <version>6-SNAPSHOT</version>
+
+ <dependencies>
+
+ <!-- provided -->
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>controller-api</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-dev</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jdisc_http_service</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>zkfacade</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-jersey2</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>serviceview</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-provisioning</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <classifier>no_aop</classifier>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- compile -->
+
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-model-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.6</version>
+ </dependency>
+
+ <!-- test -->
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>application</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>testutil</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.easytesting</groupId>
+ <artifactId>fest-assert</artifactId>
+ <version>1.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>bundle-plugin</artifactId>
+ <configuration>
+ <useCommonAssemblyIds>false</useCommonAssemblyIds>
+ <WebInfUrl>/WEB-INF/web.xml</WebInfUrl>
+ </configuration>
+ <extensions>true</extensions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:all</arg>
+ <arg>-Xlint:-serial</arg>
+ <arg>-Xlint:-deprecation</arg>
+ <arg>-Xlint:-try</arg>
+ <arg>-Werror</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-deploy.jar</file>
+ <type>jar</type>
+ <classifier>deploy</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java
new file mode 100644
index 00000000000..ffe7cb6ef67
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
+
+/**
+ * @author Tony Vaagenes
+ */
+public class AlreadyExistsException extends IllegalArgumentException {
+
+ /**
+ * Example message: Tenant 'myId' already exists.
+ *
+ * @param capitalizedType e.g. Tenant, Application
+ * @param id The id of the entity that didn't exist.
+ *
+ */
+ public AlreadyExistsException(String capitalizedType, String id) {
+ super(String.format("%s '%s' already exists", capitalizedType, id));
+ }
+
+ public AlreadyExistsException(Identifier identifier) {
+ this(identifier.capitalizedType(), identifier.id());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
new file mode 100644
index 00000000000..971438e008c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -0,0 +1,276 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * An instance of an application.
+ *
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class Application {
+
+ private final ApplicationId id;
+ private final DeploymentSpec deploymentSpec;
+ private final ValidationOverrides validationOverrides;
+ private final Map<Zone, Deployment> deployments;
+ private final DeploymentJobs deploymentJobs;
+ private final Optional<Change> deploying;
+ private final boolean outstandingChange;
+
+ /** Creates an empty application */
+ public Application(ApplicationId id) {
+ this(id, DeploymentSpec.empty, ValidationOverrides.empty, ImmutableMap.of(), new DeploymentJobs(0L),
+ Optional.empty(), false); // TODO: Get rid of the 0
+ }
+
+ /** Used from persistence layer: Do not use */
+ public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
+ List<Deployment> deployments,
+ DeploymentJobs deploymentJobs, Optional<Change> deploying, boolean outstandingChange) {
+ this(id, deploymentSpec, validationOverrides,
+ deployments.stream().collect(Collectors.toMap(d -> d.zone(), d -> d)),
+ deploymentJobs, deploying, outstandingChange);
+ }
+
+ private Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
+ Map<Zone, Deployment> deployments,
+ DeploymentJobs deploymentJobs, Optional<Change> deploying, boolean outstandingChange) {
+ Objects.requireNonNull(id, "id cannot be null");
+ Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
+ Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null");
+ Objects.requireNonNull(deployments, "deployments cannot be null");
+ Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null");
+ Objects.requireNonNull(deploying, "deploying cannot be null");
+ this.id = id;
+ this.deploymentSpec = deploymentSpec;
+ this.validationOverrides = validationOverrides;
+ this.deployments = ImmutableMap.copyOf(deployments);
+ this.deploymentJobs = deploymentJobs;
+ this.deploying = deploying;
+ this.outstandingChange = outstandingChange;
+ }
+
+ public ApplicationId id() { return id; }
+
+ /**
+ * Returns the last deployed deployment spec of this application,
+ * or the empty deployment spec if it has never been deployed
+ */
+ public DeploymentSpec deploymentSpec() { return deploymentSpec; }
+
+ /**
+ * Returns the last deployed validation overrides of this application,
+ * or the empty validation overrides if it has never been deployed
+ * (or was deployed with an empty/missing validation overrides)
+ */
+ public ValidationOverrides validationOverrides() { return validationOverrides; }
+
+ /** Returns an immutable map of the current deployments of this */
+ public Map<Zone, Deployment> deployments() { return deployments; }
+
+ public DeploymentJobs deploymentJobs() { return deploymentJobs; }
+
+ /**
+ * Returns the change that is currently in the process of being deployed on this application,
+ * or empty if no change is currently being deployed.
+ */
+ public Optional<Change> deploying() { return deploying; }
+
+ /**
+ * Returns whether this has an outstanding change (in the source repository), which
+ * has currently not started deploying (because a deployment is (or was) already in progress
+ */
+ public boolean hasOutstandingChange() { return outstandingChange; }
+
+ /**
+ * Returns the oldest version this has deployed in a permanent zone (not test or staging),
+ * or empty version if it is not deployed anywhere
+ */
+ public Optional<Version> deployedVersion() {
+ return deployments().values().stream()
+ .filter(deployment -> isPermanent(deployment.zone().environment()))
+ .sorted(Comparator.comparing(Deployment::version))
+ .findFirst()
+ .map(Deployment::version);
+ }
+
+ /** The version that should be used to compile this application */
+ public Version compileVersion(Controller controller) {
+ return deployedVersion().orElse(controller.systemVersion());
+ }
+
+ public Application withProjectId(long projectId) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), deploying, outstandingChange);
+ }
+
+ public Application withJiraIssueId(Optional<String> jiraIssueId) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withJiraIssueId(jiraIssueId), deploying, outstandingChange);
+ }
+
+ public Application withJobCompletion(JobReport report, Instant notificationTime, Controller controller) {
+ return new Application(id,
+ deploymentSpec,
+ validationOverrides,
+ deployments,
+ deploymentJobs.withCompletion(report, notificationTime, controller),
+ deploying,
+ outstandingChange);
+ }
+
+ public Application withJobTriggering(JobType type, Instant triggerTime, Controller controller) {
+ return new Application(id,
+ deploymentSpec,
+ validationOverrides,
+ deployments,
+ deploymentJobs.withTriggering(type,
+ determineTriggerVersion(type, controller),
+ determineTriggerRevision(type, controller),
+ triggerTime),
+ deploying,
+ outstandingChange);
+ }
+
+ public Application with(Deployment deployment) {
+ Map<Zone, Deployment> deployments = new LinkedHashMap<>(this.deployments);
+ deployments.put(deployment.zone(), deployment);
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application with(DeploymentJobs deploymentJobs) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application withoutDeploymentIn(Zone zone) {
+ Map<Zone, Deployment> deployments = new LinkedHashMap<>(this.deployments);
+ deployments.remove(zone);
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application withoutDeploymentJob(JobType jobType) {
+ DeploymentJobs deploymentJobs = this.deploymentJobs.without(jobType);
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application with(DeploymentSpec deploymentSpec) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application with(ValidationOverrides validationOverrides) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application withDeploying(Optional<Change> deploying) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ public Application withOutstandingChange(boolean outstandingChange) {
+ return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange);
+ }
+
+ private Version determineTriggerVersion(JobType jobType, Controller controller) {
+ Optional<Zone> zone = jobType.zone(controller.system());
+ if ( ! zone.isPresent()) // a sloppy test TODO: Fix
+ return controller.systemVersion();
+ return currentDeployVersion(controller, zone.get());
+ }
+
+ /** Returns the version a deployment to this zone should use for this application */
+ Version currentDeployVersion(Controller controller, Zone zone) {
+ if ( ! deploying().isPresent())
+ return currentVersion(controller, zone);
+ else if ( deploying().get() instanceof Change.ApplicationChange)
+ return currentVersion(controller, zone);
+ else
+ return ((Change.VersionChange) deploying().get()).version();
+ }
+
+ /** Returns the current version this application has, or if none; should use, in the given zone */
+ Version currentVersion(Controller controller, Zone zone) {
+ Deployment currentDeployment = deployments().get(zone);
+ if (currentDeployment != null) // Already deployed in this zone: Use that version
+ return currentDeployment.version();
+
+ return deployedVersion().orElse(controller.systemVersion());
+ }
+
+ private Optional<ApplicationRevision> determineTriggerRevision(JobType jobType, Controller controller) {
+ Optional<Zone> zone = jobType.zone(controller.system());
+ if ( ! zone.isPresent()) // a sloppy test TODO: Fix
+ return Optional.empty();
+ return currentDeployRevision(jobType.zone(controller.system()).get());
+ }
+
+ /** Returns the version a deployment to this zone should use for this application, or empty if we don't know */
+ Optional<ApplicationRevision> currentDeployRevision(Zone zone) {
+ if ( ! deploying().isPresent())
+ return currentRevision(zone);
+ else if ( deploying().get() instanceof Change.VersionChange)
+ return currentRevision(zone);
+ else
+ return ((Change.ApplicationChange)deploying().get()).revision();
+ }
+
+ /**
+ * Returns the current revision this application has, or if none; should use assuming no change,
+ * in the given zone. Empty if not known
+ */
+ Optional<ApplicationRevision> currentRevision(Zone zone) {
+ Deployment currentDeployment = deployments().get(zone);
+ if (currentDeployment != null) // Already deployed in this zone: Use that revision
+ return Optional.of(currentDeployment.revision());
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (! (o instanceof Application)) return false;
+
+ Application that = (Application) o;
+
+ return id.equals(that.id);
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "application '" + id + "'";
+ }
+
+ private boolean isPermanent(Environment environment) {
+ if (environment == Environment.dev) return false;
+ if (environment == Environment.perf) return false;
+ if (environment == Environment.test) return false;
+ if (environment == Environment.staging) return false;
+ return true;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
new file mode 100644
index 00000000000..51bf530ed4a
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -0,0 +1,541 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.api.ActivateResult;
+import com.yahoo.vespa.hosted.controller.api.ApplicationAlias;
+import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
+import com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer;
+import com.yahoo.vespa.hosted.controller.persistence.ControllerDb;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.rotation.RotationRepository;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Clock;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * A singleton owned by the Controller which contains the methods and state for controlling applications.
+ *
+ * @author bratseth
+ */
+public class ApplicationController {
+
+ private static final Logger log = Logger.getLogger(ApplicationController.class.getName());
+
+ /** The controller owning this */
+ private final Controller controller;
+
+ /** For permanent storage */
+ private final ControllerDb db;
+ /** For working memory storage and sharing between controllers */
+ private final CuratorDb curator;
+
+ private final RotationRepository rotationRepository;
+ private final ZmsClientFactory zmsClientFactory;
+ private final NameService nameService;
+ private final ConfigServerClient configserverClient;
+ private final RoutingGenerator routingGenerator;
+ private final Clock clock;
+
+ private final DeploymentTrigger deploymentTrigger;
+
+ ApplicationController(Controller controller, ControllerDb db, CuratorDb curator,
+ RotationRepository rotationRepository,
+ ZmsClientFactory zmsClientFactory,
+ NameService nameService, ConfigServerClient configserverClient,
+ RoutingGenerator routingGenerator, Clock clock) {
+ this.controller = controller;
+ this.db = db;
+ this.curator = curator;
+ this.rotationRepository = rotationRepository;
+ this.zmsClientFactory = zmsClientFactory;
+ this.nameService = nameService;
+ this.configserverClient = configserverClient;
+ this.routingGenerator = routingGenerator;
+ this.clock = clock;
+
+ this.deploymentTrigger = new DeploymentTrigger(controller, curator, clock);
+
+ for (Application application : db.listApplications()) {
+ try (Lock lock = lock(application.id())) {
+ Optional<Application> optionalApplication = db.getApplication(application.id()); // re-get inside lock
+ if ( ! optionalApplication.isPresent()) continue; // was removed since listing; ok
+ store(optionalApplication.get(), lock); // re-write all applications to update storage format
+ }
+ }
+ }
+
+ /** Returns the application with the given id, or null if it is not present */
+ public Optional<Application> get(ApplicationId id) {
+ return db.getApplication(id);
+ }
+
+ /**
+ * Returns the application with the given id
+ *
+ * @throws IllegalArgumentException if it does not exist
+ */
+ public Application require(ApplicationId id) {
+ return get(id).orElseThrow(() -> new IllegalArgumentException(id + " not found"));
+ }
+
+ /** Returns a snapshot of all applications */
+ public List<Application> asList() {
+ return db.listApplications();
+ }
+
+ /** Returns all applications of a tenant */
+ public List<Application> asList(TenantName tenant) {
+ return db.listApplications(new TenantId(tenant.value()));
+ }
+
+ /**
+ * Set the rotations marked as 'global' either 'in' or 'out of' service.
+ *
+ * @return The list of endpoints successfully alertered
+ * @throws IOException if rotation status cannot be updated
+ */
+ public List<String> setGlobalRotationStatus(DeploymentId deploymentId, EndpointStatus status) throws IOException {
+ List<String> rotations = new ArrayList<>();
+ for (RoutingEndpoint endpoint : routingGenerator.endpoints(deploymentId)) {
+ if (endpoint.isGlobal()) {
+ configserverClient.setGlobalRotationStatus(deploymentId, endpoint.getEndpoint(), status);
+ rotations.add(endpoint.getEndpoint());
+ }
+ }
+ return rotations;
+ }
+
+ /**
+ * Get the endpoint status for rotations marked as 'global'
+ *
+ * @return The list of endpoints successfully alertered
+ * @throws IOException if global rotation status cannot be determined
+ */
+ public Map<String, EndpointStatus> getGlobalRotationStatus(DeploymentId deploymentId) throws IOException {
+ Map<String, EndpointStatus> result = new HashMap<>();
+ for (RoutingEndpoint endpoint : routingGenerator.endpoints(deploymentId)) {
+ if (endpoint.isGlobal()) {
+ EndpointStatus status = configserverClient.getGlobalRotationStatus(deploymentId, endpoint.getEndpoint());
+ result.put(endpoint.getEndpoint(), status);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Creates a new application for an existing tenant.
+ *
+ * @throws IllegalArgumentException if the application already exists
+ */
+ public Application createApplication(ApplicationId id, Optional<NToken> token) {
+ if ( ! (id.instance().value().equals("default") || id.instance().value().startsWith("default-pr"))) // TODO: Support instances properly
+ throw new UnsupportedOperationException("Only the instance names 'default' and names starting with 'default-pr' are supported at the moment");
+ try (Lock lock = lock(id)) {
+ if (get(id).isPresent())
+ throw new IllegalArgumentException("An application with id '" + id + "' already exists");
+
+ com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value());
+
+ Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(id.tenant().value()));
+ if ( ! tenant.isPresent())
+ throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist");
+ if (get(id).isPresent())
+ throw new IllegalArgumentException("Could not create '" + id + "': Application already exists");
+ if (get(dashToUnderscore(id)).isPresent()) // VESPA-1945
+ throw new IllegalArgumentException("Could not create '" + id + "': Application " + dashToUnderscore(id) + " already exists");
+ if (tenant.get().isAthensTenant() && ! token.isPresent())
+ throw new IllegalArgumentException("Could not create '" + id + "': No NToken provided");
+ if (tenant.get().isAthensTenant()) {
+ ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get());
+ try {
+ zmsClient.deleteApplication(tenant.get().getAthensDomain().get(),
+ new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value()));
+ }
+ catch (ZmsException ignored) {
+ }
+ zmsClient.addApplication(tenant.get().getAthensDomain().get(),
+ new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value()));
+ }
+ Application application = new Application(id);
+ store(application, lock);
+ log.info("Created " + application);
+ return application;
+ }
+ }
+
+ /** Deploys an application. If the application does not exist it is created. */
+ // TODO: Get rid of the options arg
+ public ActivateResult deployApplication(ApplicationId applicationId, com.yahoo.config.provision.Zone zone,
+ ApplicationPackage applicationPackage, DeployOptions options) {
+ try (Lock lock = lock(applicationId)) {
+ // Determine what we are doing
+ Application application = get(applicationId).orElse(new Application(applicationId));
+ DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone);
+ Version version = decideVersion(application, zone, options);
+ ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob);
+
+ // Ensure that the deploying change is tested
+ // FIXME: For now only for non-self-triggering applications - VESPA-8418
+ if (!application.deploymentJobs().isSelfTriggering() && !zone.environment().isManuallyDeployed() && !application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) {
+ throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone +
+ " as pending " + application.deploying().get() +
+ " is untested");
+ }
+
+ // Don't update/store applicationpackage information when deploying previous application package (initial staging step)
+ if(! options.deployCurrentVersion) {
+ // Add missing information to application
+ application = application.with(applicationPackage.deploymentSpec());
+ application = application.with(applicationPackage.validationOverrides());
+ if (options.screwdriverBuildJob.isPresent() && options.screwdriverBuildJob.get().screwdriverId != null)
+ application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value());
+ if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange)
+ application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision)));
+ if (!triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) {
+ // Triggering information is used to store which changes were made or attempted
+ // - For self-triggered applications we don't have any trigger information, so we add it here.
+ // - For all applications, we don't have complete control over which revision is actually built,
+ // so we update it here with what we actually triggered if necessary
+ application = application.with(application.deploymentJobs().withTriggering(jobType, version, Optional.of(revision), clock.instant()));
+ }
+
+ store(application, lock); // store missing information even if we fail deployment below
+
+ // Delete zones not listed in DeploymentSpec, if allowed
+ // We do this at deployment time to be able to return a validation failure message when necessary
+ application = deleteRemovedDeployments(application);
+
+ // Clean up deployment jobs that are no longer referenced by deployment spec
+ application = deleteUnreferencedDeploymentJobs(application);
+ }
+
+ // Carry out deployment
+ DeploymentId deploymentId = new DeploymentId(applicationId, zone);
+ ApplicationRotation rotationInDns = registerRotationInDns(deploymentId, getOrAssignRotation(deploymentId,
+ applicationPackage));
+ options = withVersion(version, options);
+ ConfigServerClient.PreparedApplication preparedApplication =
+ configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), applicationPackage.zippedContent());
+ preparedApplication.activate();
+ application = application.with(new Deployment(zone, revision, version, clock.instant()));
+ store(application, lock);
+
+ return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.messages(), preparedApplication.prepareResponse());
+ }
+ }
+
+ private Version decideVersion(Application application, Zone zone, DeployOptions options) {
+ if (options.deployCurrentVersion)
+ return application.currentVersion(controller, zone);
+
+ if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide
+ return options.vespaVersion.map(Version::new).orElse(controller.systemVersion());
+
+ if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed())
+ throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone +
+ " as a deployment is not currently expected");
+
+ return application.currentDeployVersion(controller, zone);
+ }
+
+ private Application deleteRemovedDeployments(Application application) {
+ List<Deployment> deploymentsToRemove = application.deployments().values().stream()
+ .filter(deployment -> deployment.zone().environment() == Environment.prod)
+ .filter(deployment -> ! application.deploymentSpec().includes(deployment.zone().environment(),
+ Optional.of(deployment.zone().region())))
+ .collect(Collectors.toList());
+
+ if (deploymentsToRemove.isEmpty()) return application;
+
+ if ( ! application.validationOverrides().allows(ValidationId.deploymentRemoval, clock.instant()))
+ throw new IllegalArgumentException(ValidationId.deploymentRemoval.value() + ": " + application +
+ " is deployed in " +
+ deploymentsToRemove.stream()
+ .map(deployment -> deployment.zone().region().value())
+ .collect(Collectors.joining(", ")) +
+ ", but does not include " +
+ (deploymentsToRemove.size() > 1 ? "these zones" : "this zone") +
+ " in deployment.xml");
+
+ Application applicationWithRemoval = application;
+ for (Deployment deployment : deploymentsToRemove)
+ applicationWithRemoval = deactivate(applicationWithRemoval, deployment, false);
+ return applicationWithRemoval;
+ }
+
+ private Application deleteUnreferencedDeploymentJobs(Application application) {
+ for (DeploymentJobs.JobType job : application.deploymentJobs().jobStatus().keySet()) {
+ if (!job.isProduction()) {
+ continue;
+ }
+ Optional<Zone> zone = job.zone(controller.system());
+ if (!zone.isPresent()) {
+ continue;
+ }
+ if (!application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region))) {
+ application = application.withoutDeploymentJob(job);
+ }
+ }
+ return application;
+ }
+
+ private boolean triggeredWith(ApplicationRevision revision, Application application, DeploymentJobs.JobType jobType) {
+ if (jobType == null) return false;
+ JobStatus status = application.deploymentJobs().jobStatus().get(jobType);
+ if (status == null) return false;
+ if ( ! status.lastTriggered().isPresent()) return false;
+ JobStatus.JobRun triggered = status.lastTriggered().get();
+ if ( ! triggered.revision().isPresent()) return false;
+ return triggered.revision().get().equals(revision);
+ }
+
+ private DeployOptions withVersion(Version version, DeployOptions options) {
+ return new DeployOptions(options.screwdriverBuildJob,
+ Optional.of(version),
+ options.ignoreValidationErrors,
+ options.deployCurrentVersion);
+ }
+
+ private ApplicationRevision toApplicationPackageRevision(ApplicationPackage applicationPackage,
+ Optional<ScrewdriverBuildJob> screwDriverBuildJob) {
+ if ( ! screwDriverBuildJob.isPresent())
+ return ApplicationRevision.from(applicationPackage.hash());
+
+ GitRevision gitRevision = screwDriverBuildJob.get().gitRevision;
+ if (gitRevision.repository == null || gitRevision.branch == null || gitRevision.commit == null)
+ return ApplicationRevision.from(applicationPackage.hash());
+
+ return ApplicationRevision.from(applicationPackage.hash(), new SourceRevision(gitRevision.repository.id(),
+ gitRevision.branch.id(),
+ gitRevision.commit.id()));
+ }
+
+ private ApplicationRotation registerRotationInDns(DeploymentId deploymentId, ApplicationRotation applicationRotation) {
+ ApplicationAlias alias = new ApplicationAlias(deploymentId.applicationId());
+ if (applicationRotation.rotations().isEmpty()) return applicationRotation;
+
+ Rotation rotation = applicationRotation.rotations().iterator().next(); // at this time there should be only one rotation assigned
+ String endpointName = alias.toString();
+ try {
+ Optional<Record> record = nameService.findRecord(Record.Type.CNAME, rotation.rotationName);
+ if (!record.isPresent()) {
+ RecordId recordId = nameService.createCname(endpointName, rotation.rotationName);
+ log.info("Registered mapping with record ID " + recordId.id() + ": " +
+ endpointName + " -> " + rotation.rotationName);
+ }
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Failed to register CNAME", e);
+ }
+ return new ApplicationRotation(Collections.singleton(endpointName), Collections.singleton(rotation));
+ }
+
+ private ApplicationRotation getOrAssignRotation(DeploymentId deploymentId, ApplicationPackage applicationPackage) {
+ if (deploymentId.zone().environment().equals(Environment.prod)) {
+ return new ApplicationRotation(Collections.emptySet(),
+ rotationRepository.getOrAssignRotation(deploymentId.applicationId(),
+ applicationPackage.deploymentSpec()));
+ } else {
+ return new ApplicationRotation(Collections.emptySet(),
+ Collections.emptySet());
+ }
+ }
+
+ /** Returns the endpoints of the deployment, or empty if obtaining them failed */
+ public Optional<InstanceEndpoints> getDeploymentEndpoints(DeploymentId deploymentId) {
+ try {
+ List<RoutingEndpoint> endpoints = routingGenerator.endpoints(deploymentId);
+ List<URI> endPointUrls = new ArrayList<>();
+ for (RoutingEndpoint endpoint : endpoints) {
+ try {
+ endPointUrls.add(new URI(endpoint.getEndpoint()));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Routing generator returned illegal url's", e);
+ }
+ }
+ return Optional.of(new InstanceEndpoints(endPointUrls));
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e);
+ return Optional.empty();
+ }
+ }
+
+ /**
+ * Deletes the application with this id
+ *
+ * @return the deleted application, or null if it did not exist
+ * @throws IllegalArgumentException if the application has deployments or the caller is not authorized
+ */
+ public Application deleteApplication(ApplicationId id, Optional<NToken> token) {
+ try (Lock lock = lock(id)) {
+ Optional<Application> application = get(id);
+ if ( ! application.isPresent()) return null;
+ if ( ! application.get().deployments().isEmpty())
+ throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments");
+
+ Tenant tenant = controller.tenants().tenant(new TenantId(id.tenant().value())).get();
+ if (tenant.isAthensTenant() && ! token.isPresent())
+ throw new IllegalArgumentException("Could not delete '" + application + "': No NToken provided");
+
+ // NB: Next 2 lines should have been one transaction
+ if (tenant.isAthensTenant())
+ zmsClientFactory.createClientWithAuthorizedServiceToken(token.get())
+ .deleteApplication(tenant.getAthensDomain().get(), new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value()));
+ db.deleteApplication(id);
+
+ log.info("Deleted " + application.get());
+ return application.get();
+ }
+ }
+
+ public void setJiraIssueId(ApplicationId id, Optional<String> jiraIssueId) {
+ try (Lock lock = lock(id)) {
+ get(id).ifPresent(application -> store(application.withJiraIssueId(jiraIssueId), lock));
+ }
+ }
+
+ /**
+ * Replace any previous version of this application by this instance
+ *
+ * @param application the application version to store
+ * @param lock the lock held on this application since before modification started
+ */
+ @SuppressWarnings("unused") // lock is part of the signature to remind people to acquire it, not needed internally
+ public void store(Application application, Lock lock) {
+ db.store(application);
+ }
+
+ public void notifyJobCompletion(JobReport report) {
+ if ( ! get(report.applicationId()).isPresent()) {
+ log.log(Level.WARNING, "Ignoring completion of job of project '" + report.projectId() +
+ "': Unknown application '" + report.applicationId() + "'");
+ return;
+ }
+ deploymentTrigger.triggerFromCompletion(report);
+ }
+
+ // TODO: Collapse this method and the next
+ public void restart(DeploymentId deploymentId) {
+ try {
+ configserverClient.restart(deploymentId, Optional.empty());
+ }
+ catch (NoInstanceException e) {
+ throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment");
+ }
+ }
+ public void restartHost(DeploymentId deploymentId, Hostname hostname) {
+ try {
+ configserverClient.restart(deploymentId, Optional.of(hostname));
+ }
+ catch (NoInstanceException e) {
+ throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment");
+ }
+ }
+
+ public Application deactivate(Application application, Deployment deployment, boolean requireThatDeploymentHasExpired) {
+ try (Lock lock = lock(application.id())) {
+ // TODO: ignore no application errors for config server client,
+ // only return such errors from sherpa client.
+ if (requireThatDeploymentHasExpired && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment,
+ clock.instant()))
+ return application;
+
+ try {
+ configserverClient.deactivate(new DeploymentId(application.id(), deployment.zone()));
+ }
+ catch (NoInstanceException e) {
+ // ok; already gone
+ }
+ application = application.withoutDeploymentIn(deployment.zone());
+ store(application, lock);
+ return application;
+ }
+ }
+
+ public DeploymentTrigger deploymentTrigger() { return deploymentTrigger; }
+
+ private ApplicationId dashToUnderscore(ApplicationId id) {
+ return ApplicationId.from(id.tenant().value(),
+ id.application().value().replaceAll("-", "_"),
+ id.instance().value());
+ }
+
+ public ConfigServerClient configserverClient() { return configserverClient; }
+
+ /**
+ * Returns a lock which provides exclusive rights to changing this application.
+ * Any operation which stores an application need to first acquire this lock, then read, modify
+ * and store the application, and finally release (close) the lock.
+ */
+ public Lock lock(ApplicationId application) {
+ return curator.lock(application, Duration.ofMinutes(10));
+ }
+
+ private static final class ApplicationRotation {
+
+ private final ImmutableSet<String> cnames;
+ private final ImmutableSet<Rotation> rotations;
+
+ public ApplicationRotation(Set<String> cnames, Set<Rotation> rotations) {
+ this.cnames = ImmutableSet.copyOf(cnames);
+ this.rotations = ImmutableSet.copyOf(rotations);
+ }
+
+ public Set<String> cnames() { return cnames; }
+ public Set<Rotation> rotations() { return rotations; }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
new file mode 100644
index 00000000000..dcb54f13e4b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -0,0 +1,273 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.Cost;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService;
+import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub;
+import com.yahoo.vespa.hosted.controller.api.integration.jira.Jira;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException;
+import com.yahoo.vespa.hosted.controller.persistence.ControllerDb;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import com.yahoo.vespa.hosted.rotation.RotationRepository;
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+
+import java.net.URI;
+import java.time.Clock;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+/**
+ * API to the controller. This contains (currently: should contain) the object model of everything the
+ * controller cares about, mainly tenants and applications.
+ *
+ * As the controller runtime and Controller object are singletons, this instance can read from the object model
+ * in memory. However, all changes to the object model must be persisted in the controller db.
+ *
+ * All the individual model objects reachable from the Controller are immutable.
+ *
+ * Access to the controller is multithread safe, provided the locking methods are
+ * used when accessing, modifying and storing objects provided by the controller.
+ *
+ * @author bratseth
+ */
+public class Controller extends AbstractComponent {
+
+ private static final Logger log = Logger.getLogger(Controller.class.getName());
+
+ private final CuratorDb curator;
+ private final ApplicationController applicationController;
+ private final TenantController tenantController;
+
+ /**
+ * Status of Vespa versions across the system.
+ * This is expensive to maintain so that is done periodically by a maintenance job
+ */
+ private final AtomicReference<VersionStatus> versionStatus;
+
+ private final Clock clock;
+
+ private final RotationRepository rotationRepository;
+ private final GitHub gitHub;
+ private final EntityService entityService;
+ private final GlobalRoutingService globalRoutingService;
+ private final ZoneRegistry zoneRegistry;
+ private final Cost cost;
+ private final ConfigServerClient configServerClient;
+ private final MetricsService metricsService;
+ private final Chef chefClient;
+ private final Athens athens;
+
+ /**
+ * Creates a controller
+ *
+ * @param db the db storing persistent state
+ * @param curator the curator instance storing working state shared between controller instances
+ */
+ @Inject
+ public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository,
+ GitHub gitHub, Jira jiraClient, EntityService entityService,
+ GlobalRoutingService globalRoutingService,
+ ZoneRegistry zoneRegistry, Cost cost, ConfigServerClient configServerClient,
+ MetricsService metricsService, NameService nameService,
+ RoutingGenerator routingGenerator, Chef chefClient, Athens athens) {
+ this(db, curator, rotationRepository,
+ gitHub, jiraClient, entityService, globalRoutingService, zoneRegistry,
+ cost, configServerClient, metricsService, nameService, routingGenerator, chefClient,
+ Clock.systemUTC(), athens);
+ }
+
+ public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository,
+ GitHub gitHub, Jira jiraClient, EntityService entityService,
+ GlobalRoutingService globalRoutingService,
+ ZoneRegistry zoneRegistry, Cost cost, ConfigServerClient configServerClient,
+ MetricsService metricsService, NameService nameService,
+ RoutingGenerator routingGenerator, Chef chefClient, Clock clock, Athens athens) {
+ Objects.requireNonNull(db, "Controller db cannot be null");
+ Objects.requireNonNull(curator, "Curator cannot be null");
+ Objects.requireNonNull(rotationRepository, "Rotation repository cannot be null");
+ Objects.requireNonNull(gitHub, "GitHubClient cannot be null");
+ Objects.requireNonNull(jiraClient, "JiraClient cannot be null");
+ Objects.requireNonNull(entityService, "EntityService cannot be null");
+ Objects.requireNonNull(globalRoutingService, "GlobalRoutingService cannot be null");
+ Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null");
+ Objects.requireNonNull(cost, "Cost cannot be null");
+ Objects.requireNonNull(configServerClient, "ConfigServerClient cannot be null");
+ Objects.requireNonNull(metricsService, "MetricsService cannot be null");
+ Objects.requireNonNull(nameService, "NameService cannot be null");
+ Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null");
+ Objects.requireNonNull(chefClient, "ChefClient cannot be null");
+ Objects.requireNonNull(clock, "Clock cannot be null");
+ Objects.requireNonNull(athens, "Athens cannot be null");
+
+ this.rotationRepository = rotationRepository;
+ this.curator = curator;
+ this.gitHub = gitHub;
+ this.entityService = entityService;
+ this.globalRoutingService = globalRoutingService;
+ this.zoneRegistry = zoneRegistry;
+ this.cost = cost;
+ this.configServerClient = configServerClient;
+ this.metricsService = metricsService;
+ this.chefClient = chefClient;
+ this.clock = clock;
+ this.athens = athens;
+
+ applicationController = new ApplicationController(this, db, curator, rotationRepository, athens.zmsClientFactory(),
+ nameService, configServerClient, routingGenerator, clock);
+ tenantController = new TenantController(this, db, curator, entityService);
+ versionStatus = new AtomicReference<>(VersionStatus.empty());
+ }
+
+ /** Returns the instance controlling tenants */
+ public TenantController tenants() { return tenantController; }
+
+ /** Returns the instance controlling applications */
+ public ApplicationController applications() { return applicationController; }
+
+ public List<AthensDomain> getDomainList(String prefix) {
+ return athens.unauthorizedZmsClient().getDomainList(prefix);
+ }
+
+ public Athens athens() {
+ return athens;
+ }
+
+ /**
+ * Fetch list of all active OpsDB properties.
+ *
+ * @return Hashed map with the property ID as key and property name as value
+ */
+ public Map<PropertyId, Property> fetchPropertyList() {
+ return entityService.listProperties();
+ }
+
+ public Clock clock() { return clock; }
+
+ public ApplicationCost getApplicationCost(com.yahoo.config.provision.ApplicationId application,
+ com.yahoo.config.provision.Zone zone)
+ throws NotFoundCheckedException {
+ return cost.getApplicationCost(zone.environment(), zone.region(), application);
+ }
+
+ public URI getElkUri(Environment environment, RegionName region, DeploymentId deploymentId) {
+ return elkUrl(zoneRegistry.getLogServerUri(environment, region), deploymentId);
+ }
+
+ public List<URI> getConfigServerUris(Environment environment, RegionName region) {
+ return zoneRegistry.getConfigServerUris(environment, region);
+ }
+
+ public ZoneRegistry zoneRegistry() { return zoneRegistry; }
+
+ private URI elkUrl(Optional<URI> kibanaHost, DeploymentId deploymentId) {
+ String kibanaQuery = "/#/discover?_g=()&_a=(columns:!(_source)," +
+ "index:'logstash-*',interval:auto," +
+ "query:(query_string:(analyze_wildcard:!t,query:'" +
+ "HV-tenant:%22" + deploymentId.applicationId().tenant().value() + "%22%20" +
+ "AND%20HV-application:%22" + deploymentId.applicationId().application().value() + "%22%20" +
+ "AND%20HV-region:%22" + deploymentId.zone().region().value() + "%22%20" +
+ "AND%20HV-instance:%22" + deploymentId.applicationId().instance().value() + "%22%20" +
+ "AND%20HV-environment:%22" + deploymentId.zone().environment().value() + "%22'))," +
+ "sort:!('@timestamp',desc))";
+
+ URI kibanaPath = URI.create(kibanaQuery);
+ if (kibanaHost.isPresent()) {
+ return kibanaHost.get().resolve(kibanaPath);
+ } else {
+ return null;
+ }
+ }
+
+ public Set<URI> getRotationUris(ApplicationId id) {
+ return rotationRepository.getRotationUris(id);
+ }
+
+ public Map<String, RotationStatus> getHealthStatus(String hostname) {
+ return globalRoutingService.getHealthStatus(hostname);
+ }
+
+ // TODO: Model the response properly
+ public JsonNode waitForConfigConvergence(DeploymentId deploymentId, long timeout) {
+ return configServerClient.waitForConfigConverge(deploymentId, timeout);
+ }
+
+ public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) {
+ return configServerClient.getApplicationView(tenantName, applicationName, instanceName, environment, region);
+ }
+
+ // TODO: Model the response properly
+ public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath) {
+ return configServerClient.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath);
+ }
+
+ // TODO: Model the response properly
+ // TODO: What is this
+ public JsonNode grabLog(DeploymentId deploymentId) {
+ return configServerClient.grabLog(deploymentId);
+ }
+
+ public GitHub gitHub() { return gitHub; }
+
+ /** Replace the current version status by a new one */
+ public void updateVersionStatus(VersionStatus newStatus) {
+ VersionStatus currentStatus = versionStatus();
+ if (newStatus.systemVersion().isPresent() &&
+ ! newStatus.systemVersion().equals(currentStatus.systemVersion())) {
+ log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) +
+ " to " + printableVersion(newStatus.systemVersion()));
+ curator.writeSystemVersion(newStatus.systemVersion().get().versionNumber());
+ }
+
+ this.versionStatus.set(newStatus);
+ }
+
+ /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */
+ public VersionStatus versionStatus() { return versionStatus.get(); }
+
+ /** Returns the current system version: The controller should drive towards running all applications on this version */
+ public Version systemVersion() { return curator.readSystemVersion(); }
+
+ public MetricsService metricsService() { return metricsService; }
+
+ public SystemName system() {
+ return zoneRegistry.system();
+ }
+
+ public Chef chefClient() {
+ return chefClient;
+ }
+
+ private String printableVersion(Optional<VespaVersion> vespaVersion) {
+ return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("Unknown");
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java
new file mode 100644
index 00000000000..6a47957f27f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/NotExistsException.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
+
+/**
+ * An exception which indicates that a requested resource does not exist.
+ *
+ * @author Tony Vaagenes
+ */
+public class NotExistsException extends IllegalArgumentException {
+
+ public NotExistsException(String message) {
+ super(message);
+ }
+
+ /**
+ * Example message: Tenant 'myId' does not exist.
+ *
+ * @param capitalizedType e.g. Tenant, Application
+ * @param id The id of the entity that didn't exist.
+ *
+ */
+ public NotExistsException(String capitalizedType, String id) {
+ super(String.format("%s '%s' does not exist", capitalizedType, id));
+ }
+
+ public NotExistsException(Identifier id) {
+ this(id.capitalizedType(), id.id());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
new file mode 100644
index 00000000000..fafd0b04dd2
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
@@ -0,0 +1,238 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService;
+import com.yahoo.vespa.hosted.controller.persistence.ControllerDb;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.persistence.PersistenceException;
+
+import java.time.Duration;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * A singleton owned by the Controller which contains the methods and state for controlling applications.
+ *
+ * @author bratseth
+ */
+public class TenantController {
+
+ private static final Logger log = Logger.getLogger(TenantController.class.getName());
+
+ /** The controller owning this */
+ private final Controller controller;
+
+ /** For permanent storage */
+ private final ControllerDb db;
+
+ /** For working memory storage and sharing between controllers */
+ private final CuratorDb curator;
+
+ private final ZmsClientFactory zmsClientFactory;
+ private final EntityService entityService;
+
+ public TenantController(Controller controller, ControllerDb db, CuratorDb curator, EntityService entityService) {
+ this.controller = controller;
+ this.db = db;
+ this.curator = curator;
+ this.zmsClientFactory = controller.athens().zmsClientFactory();
+ this.entityService = entityService;
+ }
+
+ public List<Tenant> asList() {
+ return db.listTenants();
+ }
+
+ public List<Tenant> asList(UserId user) {
+ Set<UserGroup> userGroups = entityService.getUserGroups(user);
+ Set<AthensDomain> userDomains = new HashSet<>(zmsClientFactory.createClientWithServicePrincipal()
+ .getTenantDomainsForUser(controller.athens().principalFrom(user)));
+
+ Predicate<Tenant> hasUsersGroup = (tenant) -> tenant.getUserGroup().isPresent() && userGroups.contains(tenant.getUserGroup().get());
+ Predicate<Tenant> hasUsersDomain = (tenant) -> tenant.getAthensDomain().isPresent() && userDomains.contains(tenant.getAthensDomain().get());
+ Predicate<Tenant> isUserTenant = (tenant) -> tenant.getId().equals(user.toTenantId());
+
+ return asList().stream()
+ .filter(t -> hasUsersGroup.test(t) || hasUsersDomain.test(t) || isUserTenant.test(t))
+ .collect(Collectors.toList());
+ }
+
+ public Tenant createUserTenant(String userName) {
+ TenantId userTenantId = new UserId(userName).toTenantId();
+ try (Lock lock = lock(userTenantId)) {
+ Tenant tenant = Tenant.createUserTenant(userTenantId);
+ internalCreateTenant(tenant, Optional.empty());
+ return tenant;
+ }
+ }
+
+ /** Creates an Athens or OpsDb tenant. */
+ // TODO: Rename to createAthensTenant and move creation here when opsDbTenant creation is removed */
+ public void addTenant(Tenant tenant, Optional<NToken> token) {
+ try (Lock lock = lock(tenant.getId())) {
+ internalCreateTenant(tenant, token);
+ }
+ }
+
+ private void internalCreateTenant(Tenant tenant, Optional<NToken> token) {
+ TenantId.validate(tenant.getId().id());
+ if (tenant(tenant.getId()).isPresent())
+ throw new IllegalArgumentException("Tenant '" + tenant.getId() + "' already exists");
+ if (tenant(dashToUnderscore(tenant.getId())).isPresent())
+ throw new IllegalArgumentException("Could not create " + tenant + ": Tenant " + dashToUnderscore(tenant.getId()) + " already exists");
+ if (tenant.isAthensTenant() && ! token.isPresent())
+ throw new IllegalArgumentException("Could not create " + tenant + ": No NToken provided");
+
+ if (tenant.isAthensTenant()) {
+ AthensDomain domain = tenant.getAthensDomain().get();
+ Optional<Tenant> existingTenantWithDomain = tenantHaving(domain);
+ if (existingTenantWithDomain.isPresent())
+ throw new IllegalArgumentException("Could not create " + tenant + ": The Athens domain '" + domain +
+ "' is already connected to " + existingTenantWithDomain.get());
+ ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get());
+ try { zmsClient.deleteTenant(domain); } catch (ZmsException ignored) { }
+ zmsClient.createTenant(domain);
+ }
+ db.createTenant(tenant);
+ log.info("Created " + tenant);
+ }
+
+ /** Returns the tenant having the given Athens domain, or empty if none */
+ private Optional<Tenant> tenantHaving(AthensDomain domain) {
+ return asList().stream().filter(Tenant::isAthensTenant)
+ .filter(t -> t.getAthensDomain().get().equals(domain))
+ .findAny();
+ }
+
+ public Optional<Tenant> tenant(TenantId id) {
+ try {
+ return db.getTenant(id);
+ } catch (PersistenceException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void updateTenant(Tenant updatedTenant, Optional<NToken> token) {
+ try (Lock lock = lock(updatedTenant.getId())) {
+ if ( ! tenant(updatedTenant.getId()).isPresent())
+ throw new IllegalArgumentException("Could not update " + updatedTenant + ": Tenant does not exist");
+ if (updatedTenant.isAthensTenant() && ! token.isPresent())
+ throw new IllegalArgumentException("Could not update " + updatedTenant + ": No NToken provided");
+
+ updateAthensDomain(updatedTenant, token);
+ db.updateTenant(updatedTenant);
+ log.info("Updated " + updatedTenant);
+ } catch (PersistenceException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void updateAthensDomain(Tenant updatedTenant, Optional<NToken> token) {
+ Tenant existingTenant = tenant(updatedTenant.getId()).get();
+ if ( ! existingTenant.isAthensTenant()) return;
+
+ AthensDomain existingDomain = existingTenant.getAthensDomain().get();
+ AthensDomain newDomain = updatedTenant.getAthensDomain().get();
+ if (existingDomain.equals(newDomain)) return;
+ Optional<Tenant> existingTenantWithNewDomain = tenantHaving(newDomain);
+ if (existingTenantWithNewDomain.isPresent())
+ throw new IllegalArgumentException("Could not set domain of " + updatedTenant + " to '" + newDomain +
+ "':" + existingTenantWithNewDomain.get() + " already has this domain");
+
+ ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get());
+ zmsClient.createTenant(newDomain);
+ List<Application> applications = controller.applications().asList(TenantName.from(existingTenant.getId().id()));
+ applications.forEach(a -> zmsClient.addApplication(newDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value())));
+ applications.forEach(a -> zmsClient.deleteApplication(existingDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value())));
+ zmsClient.deleteTenant(existingDomain);
+ log.info("Updated Athens domain for " + updatedTenant + " from " + existingDomain + " to " + newDomain);
+ }
+
+ public void deleteTenant(TenantId id, Optional<NToken> token) {
+ try (Lock lock = lock(id)) {
+ if ( ! tenant(id).isPresent())
+ throw new NotExistsException(id); // TODO: Change exception and message
+ if ( ! controller.applications().asList(TenantName.from(id.id())).isEmpty())
+ throw new IllegalArgumentException("Could not delete tenant '" + id + "': This tenant has active applications");
+
+ Tenant tenant = tenant(id).get();
+ if (tenant.isAthensTenant() && ! token.isPresent())
+ throw new IllegalArgumentException("Could not delete tenant '" + id + "': No NToken provided");
+
+ try {
+ db.deleteTenant(id);
+ } catch (PersistenceException e) { // TODO: Don't allow these to leak out
+ throw new RuntimeException(e);
+ }
+ if (tenant.isAthensTenant())
+ zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()).deleteTenant(tenant.getAthensDomain().get());
+ log.info("Deleted " + tenant);
+ }
+ }
+
+ public Tenant migrateTenantToAthens(TenantId tenantId,
+ AthensDomain tenantDomain,
+ PropertyId propertyId,
+ Property property,
+ NToken nToken) {
+ try (Lock lock = lock(tenantId)) {
+ Tenant existing = tenant(tenantId).orElseThrow(() -> new NotExistsException(tenantId));
+ if (existing.isAthensTenant()) return existing; // nothing to do
+ if (tenantHaving(tenantDomain).isPresent())
+ throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " +
+ "This domain is already used by " + tenantHaving(tenantDomain).get());
+ if ( ! existing.isOpsDbTenant())
+ throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " +
+ "Tenant is not currently an OpsDb tenant");
+
+ ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(nToken);
+ zmsClient.createTenant(tenantDomain);
+ List<Application> applications = controller.applications().asList(TenantName.from(existing.getId().id()));
+ applications.forEach(a -> {
+ ApplicationId applicationId = new ApplicationId(a.id().application().value());
+ zmsClient.addApplication(tenantDomain, applicationId);
+ });
+ db.deleteTenant(tenantId);
+ Tenant tenant = Tenant.createAthensTenant(tenantId, tenantDomain, property, Optional.of(propertyId));
+ db.createTenant(tenant);
+ log.info("Migrated " + existing + " to Athens using " + tenantDomain);
+ return tenant;
+ }
+ catch (PersistenceException e) {
+ throw new RuntimeException("Failed migrating " + tenantId + " to Athens", e);
+ }
+ }
+
+ private TenantId dashToUnderscore(TenantId id) {
+ return new TenantId(id.id().replaceAll("-", "_"));
+ }
+
+ /**
+ * Returns a lock which provides exclusive rights to changing this tenant.
+ * Any operation which stores a tenant need to first acquire this lock, then read, modify
+ * and store the tenant, and finally release (close) the lock.
+ */
+ private Lock lock(TenantId tenant) {
+ return curator.lock(tenant, Duration.ofMinutes(10));
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java
new file mode 100644
index 00000000000..1fb6a4a8582
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java
@@ -0,0 +1,37 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api;
+
+import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
+
+import java.util.List;
+
+/**
+ * @author Oyvind Gronnesby
+ */
+public class ActivateResult {
+
+ private final RevisionId revisionId;
+ private final List<Log> messages;
+ private final PrepareResponse prepareResponse;
+
+ public ActivateResult(RevisionId revisionId, List<Log> messages, PrepareResponse prepareResponse) {
+ this.revisionId = revisionId;
+ this.messages = messages;
+ this.prepareResponse = prepareResponse;
+ }
+
+ public RevisionId getRevisionId() {
+ return revisionId;
+ }
+
+ public List<Log> getMessages() {
+ return messages;
+ }
+
+ public PrepareResponse getPrepareResponse() {
+ return prepareResponse;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java
new file mode 100644
index 00000000000..a9e144a3227
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java
@@ -0,0 +1,57 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api;
+
+import com.yahoo.config.provision.ApplicationId;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * A DNS alias for an application endpoint.
+ *
+ * @author smorgrav
+ */
+public class ApplicationAlias {
+
+ private static final String dnsSuffix = "global.vespa.yahooapis.com";
+
+ private final ApplicationId applicationId;
+
+ public ApplicationAlias(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s.%s.%s",
+ toDns(applicationId.application().value()),
+ toDns(applicationId.tenant().value()),
+ dnsSuffix);
+ }
+
+ private String toDns(String id) {
+ return id.replace('_', '-');
+ }
+
+ public URI toHttpUri() {
+ try {
+ return new URI("http://" + this + ":4080/");
+ } catch(URISyntaxException use) {
+ throw new RuntimeException("Illegal URI syntax");
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ApplicationAlias that = (ApplicationAlias) o;
+
+ return applicationId.equals(that.applicationId);
+ }
+
+ @Override
+ public int hashCode() { return applicationId.hashCode(); }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java
new file mode 100644
index 00000000000..b9ed439eb8b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/InstanceEndpoints.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api;
+
+import java.net.URI;
+import java.util.List;
+
+/**
+ * @author Tony Vaagenes
+ */
+public class InstanceEndpoints {
+
+ private final List<URI> containerEndpoints;
+
+ public InstanceEndpoints(List<URI> containerEndpoints) {
+ this.containerEndpoints = containerEndpoints;
+ }
+
+ public List<URI> getContainerEndpoints() {
+ return containerEndpoints;
+ }
+}
+
+
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java
new file mode 100644
index 00000000000..325c40c24c8
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/Tenant.java
@@ -0,0 +1,147 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api;
+
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.TenantType;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+
+import java.util.Optional;
+
+/**
+ * @author smorgrav
+ */
+// TODO: Move this and everything it owns to com.yahoo.hosted.controller.Tenant and com.yahoo.hosted.controller.tenant.*
+public class Tenant {
+
+ private final TenantId id;
+ private final Optional<UserGroup> userGroup;
+ private final Optional<Property> property;
+ private final Optional<AthensDomain> athensDomain;
+ private final Optional<PropertyId> propertyId;
+
+ // TODO: Use factory methods. They're down at the bottom!
+ public Tenant(TenantId id, Optional<UserGroup> userGroup, Optional<Property> property, Optional<AthensDomain> athensDomain) {
+ this(id, userGroup, property, athensDomain, Optional.empty());
+ }
+
+ public Tenant(TenantId id, Optional<UserGroup> userGroup, Optional<Property> property, Optional<AthensDomain> athensDomain, Optional<PropertyId> propertyId) {
+ if (id.isUser()) {
+ require(!userGroup.isPresent(), "User tenant '%s' cannot have a user group.", id);
+ require(!property.isPresent(), "User tenant '%s' cannot have a property.", id);
+ require(!propertyId.isPresent(), "User tenant '%s' cannot have a property ID.", id);
+ require(!athensDomain.isPresent(), "User tenant '%s' cannot have an athens domain.", id);
+ } else if (athensDomain.isPresent()) {
+ require(property.isPresent(), "Athens tenant '%s' must have a property.", id);
+ require(!userGroup.isPresent(), "Athens tenant '%s' cannot have a user group.", id);
+ require(athensDomain.isPresent(), "Athens tenant '%s' must have an athens domain.", id);
+ } else {
+ require(property.isPresent(), "OpsDB tenant '%s' must have a property.", id);
+ require(userGroup.isPresent(), "OpsDb tenant '%s' must have a user group.", id);
+ require(!athensDomain.isPresent(), "OpsDb tenant '%s' cannot have an athens domain.", id);
+ }
+ this.id = id;
+ this.userGroup = userGroup;
+ this.property = property;
+ this.athensDomain = athensDomain;
+ this.propertyId = propertyId; // TODO: Check validity after TODO@14. OpsDb tenants have this set in Sherpa, while athens tenants do not.
+ }
+
+ public boolean isAthensTenant() { return athensDomain.isPresent(); }
+ public boolean isOpsDbTenant() { return userGroup.isPresent();}
+
+ public TenantType tenantType() {
+ if (athensDomain.isPresent()) {
+ return TenantType.ATHENS;
+ } else if (id.isUser()) {
+ return TenantType.USER;
+ } else {
+ return TenantType.OPSDB;
+ }
+ }
+
+ public TenantId getId() {
+ return id;
+ }
+
+ public Optional<UserGroup> getUserGroup() {
+ return userGroup;
+ }
+
+ /** OpsDB property name of the tenant, or Optional.empty() if none is stored. */
+ public Optional<Property> getProperty() {
+ return property;
+ }
+
+ /** OpsDB property ID of the tenant. Not (yet) required, so returns Optional.empty() if none is stored. */
+ public Optional<PropertyId> getPropertyId() {
+ return propertyId;
+ }
+
+ public Optional<AthensDomain> getAthensDomain() {
+ return athensDomain;
+ }
+
+ private void require(boolean statement, String message, TenantId id) {
+ if (!statement) throw new IllegalArgumentException(String.format(message, id));
+ }
+
+ public static Tenant createAthensTenant(TenantId id, AthensDomain athensDomain, Property property, Optional<PropertyId> propertyId) {
+ if (id.isUser()) {
+ throw new IllegalArgumentException("Invalid id for non-user tenant: " + id);
+ }
+ return new Tenant(id, Optional.empty(), Optional.ofNullable(property),
+ Optional.ofNullable(athensDomain), propertyId);
+ }
+
+ public static Tenant createOpsDbTenant(TenantId id, UserGroup userGroup, Property property, Optional<PropertyId> propertyId) {
+ if (id.isUser()) {
+ throw new IllegalArgumentException("Invalid id for non-user tenant: " + id);
+ }
+ return new Tenant(id, Optional.ofNullable(userGroup), Optional.ofNullable(property), Optional.empty(), propertyId);
+ }
+
+ public static Tenant createOpsDbTenant(TenantId id, UserGroup userGroup, Property property) {
+ return createOpsDbTenant(id, userGroup, property, Optional.empty());
+ }
+
+ public static Tenant createUserTenant(TenantId id) {
+ if (!id.isUser()) {
+ throw new IllegalArgumentException("Invalid id for user tenant: " + id);
+ }
+ return new Tenant(id, Optional.empty(), Optional.empty(), Optional.empty());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Tenant tenant = (Tenant) o;
+
+ if (!id.equals(tenant.id)) return false;
+ if (!userGroup.equals(tenant.userGroup)) return false;
+ if (!property.equals(tenant.property)) return false;
+ if (!athensDomain.equals(tenant.athensDomain)) return false;
+ if (!propertyId.equals(tenant.propertyId)) return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id.hashCode();
+ result = 31 * result + userGroup.hashCode();
+ result = 31 * result + property.hashCode();
+ result = 31 * result + athensDomain.hashCode();
+ result = 31 * result + propertyId.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "tenant '" + id + "'";
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java
new file mode 100644
index 00000000000..4b405f55e10
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author Tony Vaagenes
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.api;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
new file mode 100644
index 00000000000..3fcd285e0fc
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -0,0 +1,200 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
+
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/**
+ * A list of applications which can be filtered in various ways.
+ *
+ * @author bratseth
+ */
+public class ApplicationList {
+
+ private final ImmutableList<Application> list;
+
+ private ApplicationList(List<Application> applications) {
+ this.list = ImmutableList.copyOf(applications);
+ }
+
+ // ----------------------------------- Factories
+
+ public static ApplicationList from(List<Application> applications) {
+ return new ApplicationList(applications);
+ }
+
+ public static ApplicationList from(List<ApplicationId> ids, ApplicationController applications) {
+ return listOf(ids.stream().map(applications::require));
+ }
+
+ // ----------------------------------- Accessors
+
+ /** Returns the applications in this as an immutable list */
+ public List<Application> asList() { return list; }
+
+ public boolean isEmpty() { return list.isEmpty(); }
+
+ public int size() { return list.size(); }
+
+ // ----------------------------------- Filters
+
+ /** Returns the subset of applications which is currently upgrading to the given version */
+ public ApplicationList upgradingTo(Version version) {
+ return listOf(list.stream().filter(application -> isUpgradingTo(version, application)));
+ }
+
+ /** Returns the subset of applications which is currently upgrading to a version lower than the given version */
+ public ApplicationList upgradingToLowerThan(Version version) {
+ return listOf(list.stream().filter(application -> isUpgradingToLowerThan(version, application)));
+ }
+
+ /** Returns the subset of applications which is currently not upgrading to the given version */
+ public ApplicationList notUpgradingTo(Version version) {
+ return listOf(list.stream().filter(application -> ! isUpgradingTo(version, application)));
+ }
+
+ /** Returns the subset of applications which is currently not deploying a new application revision */
+ public ApplicationList notDeployingApplication() {
+ return listOf(list.stream().filter(application -> ! isDeployingApplicationChange(application)));
+ }
+
+ /** Returns the subset of applications which currently does not have any failing jobs */
+ public ApplicationList notFailing() {
+ return listOf(list.stream().filter(application -> ! application.deploymentJobs().hasFailures()));
+ }
+
+ /** Returns the subset of applications which currently does not have any failing jobs on the given version */
+ public ApplicationList notFailingOn(Version version) {
+ return listOf(list.stream().filter(application -> ! failingOn(version, application)));
+ }
+
+ /** Returns the subset of applications which have one or more deployment jobs failing for the current change */
+ public ApplicationList hasDeploymentFailures() {
+ return listOf(list.stream().filter(application -> application.deploying().isPresent() && application.deploymentJobs().failingOn(application.deploying().get())));
+ }
+
+ /** Returns the subset of applications which have at least one deployment */
+ public ApplicationList hasDeployment() {
+ return listOf(list.stream().filter(a -> !a.deployments().isEmpty()));
+ }
+
+ /** Returns the subset of applications that are currently deploying a change */
+ public ApplicationList isDeploying() {
+ return listOf(list.stream().filter(application -> application.deploying().isPresent()));
+ }
+
+ /** Returns the subset of applications which started failing after the given instant */
+ public ApplicationList startedFailingAfter(Instant instant) {
+ return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant)));
+ }
+
+ /** Returns the subset of applications which has the given upgrade policy */
+ public ApplicationList with(UpgradePolicy policy) {
+ return listOf(list.stream().filter(a -> a.deploymentSpec().upgradePolicy() == policy));
+ }
+
+ /** Returns the subset of applications which does not have the given upgrade policy */
+ public ApplicationList without(UpgradePolicy policy) {
+ return listOf(list.stream().filter(a -> a.deploymentSpec().upgradePolicy() != policy));
+ }
+
+ /** Returns the subset of applications which have at least one deployment on a lower version than the given one */
+ public ApplicationList onLowerVersionThan(Version version) {
+ return listOf(list.stream()
+ .filter(a -> a.deployments().values().stream().anyMatch(d -> d.version().isBefore(version))));
+ }
+
+ /**
+ * Returns the subset of applications which are not pull requests:
+ * Pull requests changes the application instance name to default-pr[pull-request-number]
+ */
+ public ApplicationList notPullRequest() {
+ return listOf(list.stream().filter(a -> ! a.id().instance().value().startsWith("default-pr")));
+ }
+
+ // ----------------------------------- Sorting
+
+ /**
+ * Returns this list sorted by increasing deployed version.
+ * If multiple versions are deployed the oldest is used.
+ * Applications without any deployments are ordered first.
+ */
+ public ApplicationList byIncreasingDeployedVersion() {
+ return listOf(list.stream().sorted(Comparator.comparing(application -> application.deployedVersion().orElse(Version.emptyVersion))));
+ }
+
+ /** Returns the subset of applications which currently do not have any job in progress for the given change */
+ public ApplicationList notRunningJobFor(Change.VersionChange change) {
+ return listOf(list.stream().filter(a -> !hasRunningJob(a, change)));
+ }
+
+ /** Returns the subset of applications which currently do not have any job in progress */
+ public ApplicationList notRunningJob() {
+ return listOf(list.stream().filter(a -> !a.deploymentJobs().inProgress()));
+ }
+
+ /** Returns the subset of applications which has a job that started running before the given instant */
+ public ApplicationList jobRunningSince(Instant instant) {
+ return listOf(list.stream().filter(a -> a.deploymentJobs().runningSince()
+ .map(at -> at.isBefore(instant))
+ .orElse(false)));
+ }
+
+ /** Returns the subset of applications which deploys to given environment and region */
+ public ApplicationList deploysTo(Environment environment, RegionName region) {
+ return listOf(list.stream().filter(a -> a.deploymentSpec().includes(environment, Optional.of(region))));
+ }
+
+ // ----------------------------------- Internal helpers
+
+ private static boolean isUpgradingTo(Version version, Application application) {
+ if ( ! (application.deploying().isPresent()) ) return false;
+ if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false;
+ return ((Change.VersionChange)application.deploying().get()).version().equals(version);
+ }
+
+ private static boolean isUpgradingToLowerThan(Version version, Application application) {
+ if ( ! application.deploying().isPresent()) return false;
+ if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false;
+ return ((Change.VersionChange)application.deploying().get()).version().isBefore(version);
+ }
+
+ private static boolean isDeployingApplicationChange(Application application) {
+ if ( ! application.deploying().isPresent()) return false;
+ return application.deploying().get() instanceof Change.ApplicationChange;
+ }
+
+ private static boolean failingOn(Version version, Application application) {
+ for (JobStatus jobStatus : application.deploymentJobs().jobStatus().values())
+ if ( ! jobStatus.isSuccess() && jobStatus.lastCompleted().get().version().equals(version)) return true;
+ return false;
+ }
+
+ private static boolean hasRunningJob(Application application, Change.VersionChange change) {
+ return application.deploymentJobs().jobStatus().values().stream()
+ .filter(JobStatus::inProgress)
+ .filter(jobStatus -> jobStatus.lastTriggered().isPresent())
+ .map(jobStatus -> jobStatus.lastTriggered().get())
+ .anyMatch(jobRun -> jobRun.version().equals(change.version()));
+ }
+
+ /** Convenience converter from a stream to an ApplicationList */
+ private static ApplicationList listOf(Stream<Application> applications) {
+ ImmutableList.Builder<Application> b = new ImmutableList.Builder<>();
+ applications.forEach(b::add);
+ return new ApplicationList(b.build());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
new file mode 100644
index 00000000000..6df8e901653
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
@@ -0,0 +1,75 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationOverrides;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A representation of the content of an application package.
+ * Only the deployment.xml content can be accessed as anything other than compressed data.
+ * A package is identified by a hash of the content.
+ *
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class ApplicationPackage {
+
+ private final String contentHash;
+ private final byte[] zippedContent;
+ private final DeploymentSpec deploymentSpec;
+ private final ValidationOverrides validationOverrides;
+
+ /**
+ * Creates an application package from its zipped content.
+ * This <b>assigns ownership</b> of the given byte array to this class:
+ * it must not be further changed by the caller.
+ */
+ public ApplicationPackage(byte[] zippedContent) {
+ Objects.requireNonNull(zippedContent, "The application package content cannot be null");
+ this.contentHash = DigestUtils.shaHex(zippedContent);
+ this.zippedContent = zippedContent;
+ this.deploymentSpec = extractFile("deployment.xml", zippedContent).map(DeploymentSpec::fromXml).orElse(DeploymentSpec.empty);
+ this.validationOverrides = extractFile("validation-overrides.xml", zippedContent).map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty);
+ }
+
+ /** Returns a hash of the content of this package */
+ public String hash() { return contentHash; }
+
+ /** Returns the content of this package. The content <b>must not</b> be modified. */
+ public byte[] zippedContent() { return zippedContent; }
+
+ /**
+ * Returns the deployment spec from the deployment.xml file of the package content.
+ * This is the DeploymentSpec.empty instance if this package does not contain a deployment.xml file.
+ */
+ public DeploymentSpec deploymentSpec() { return deploymentSpec; }
+
+ /**
+ * Returns the validation overrides from the validation-overrides.xml file of the package content.
+ * This is the ValidationOverrides.empty instance if this package does not contain a validation-overrides.xml file.
+ */
+ public ValidationOverrides validationOverrides() { return validationOverrides; }
+
+ private static Optional<Reader> extractFile(String fileName, byte[] zippedContent) {
+ try (ByteArrayInputStream stream = new ByteArrayInputStream(zippedContent)) {
+ ZipStreamReader reader = new ZipStreamReader(stream);
+ for (ZipStreamReader.ZipEntryWithContent entry : reader.entries())
+ if (entry.zipEntry().getName().equals(fileName) || entry.zipEntry().getName().equals("application/" + fileName)) // TODO: Remove application/ directory support
+ return Optional.of(new InputStreamReader(new ByteArrayInputStream(entry.content())));
+ return Optional.empty();
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("Exception reading application package", e);
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java
new file mode 100644
index 00000000000..1b875f28715
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRevision.java
@@ -0,0 +1,60 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * An identifier of a particular revision (exact content) of an application package,
+ * optionally with information about the source of the package revision.
+ *
+ * @author bratseth
+ */
+public class ApplicationRevision {
+
+ private final String applicationPackageHash;
+
+ private final Optional<SourceRevision> source;
+
+ private ApplicationRevision(String applicationPackageHash, Optional<SourceRevision> source) {
+ Objects.requireNonNull(applicationPackageHash, "applicationPackageHash cannot be null");
+ this.applicationPackageHash = applicationPackageHash;
+ this.source = source;
+ }
+
+ /** Create an application package revision where there is no information about its source */
+ public static ApplicationRevision from(String applicationPackageHash) {
+ return new ApplicationRevision(applicationPackageHash, Optional.empty());
+ }
+
+ /** Create an application package revision with a source */
+ public static ApplicationRevision from(String applicationPackageHash, SourceRevision source) {
+ return new ApplicationRevision(applicationPackageHash, Optional.of(source));
+ }
+
+ /** Returns a unique, content-based identifier of an application package (a hash of the content) */
+ public String id() { return applicationPackageHash; }
+
+ /**
+ * Returns information about the source of this revision, or empty if the source is not know/defined
+ * (which is the case for command-line deployment from developers, but never for deployment jobs)
+ */
+ public Optional<SourceRevision> source() { return source; }
+
+ @Override
+ public int hashCode() { return applicationPackageHash.hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if ( ! (other instanceof ApplicationRevision)) return false;
+ return this.applicationPackageHash.equals(((ApplicationRevision)other).applicationPackageHash);
+ }
+
+ @Override
+ public String toString() {
+ return "Application package revision '" + applicationPackageHash + "'" +
+ (source.isPresent() ? " with " + source.get() : "");
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
new file mode 100644
index 00000000000..596cbbebd45
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
@@ -0,0 +1,90 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.component.Version;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A change to an application
+ *
+ * @author bratseth
+ */
+public abstract class Change {
+
+ /** A change to the application package revision of an application */
+ public static class ApplicationChange extends Change {
+
+ private final Optional<ApplicationRevision> revision;
+
+ private ApplicationChange(Optional<ApplicationRevision> revision) {
+ Objects.requireNonNull(revision, "revision cannot be null");
+ this.revision = revision;
+ }
+
+ /** The revision this changes to, or empty if not known yet */
+ public Optional<ApplicationRevision> revision() { return revision; }
+
+ @Override
+ public int hashCode() { return revision.hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if ( ! (other instanceof ApplicationChange)) return false;
+ return ((ApplicationChange)other).revision.equals(this.revision);
+ }
+
+ /**
+ * Creates an application change which we don't know anything about.
+ * We are notified that a change has occurred by completion of the component job
+ * but do not get to know about what the change is until a subsequent deployment
+ * happens.
+ */
+ public static ApplicationChange unknown() {
+ return new ApplicationChange(Optional.empty());
+ }
+
+ public static ApplicationChange of(ApplicationRevision revision) {
+ return new ApplicationChange(Optional.of(revision));
+ }
+
+ @Override
+ public String toString() {
+ return "application change to " + revision.map(ApplicationRevision::toString).orElse("an unknown revision");
+ }
+
+ }
+
+ /** A change to the Vespa version running an application */
+ public static class VersionChange extends Change {
+
+ private final Version version;
+
+ public VersionChange(Version version) {
+ Objects.requireNonNull(version, "version cannot be null");
+ this.version = version;
+ }
+
+ /** The Vespa version this changes to */
+ public Version version() { return version; }
+
+ @Override
+ public int hashCode() { return version.hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if ( ! (other instanceof VersionChange)) return false;
+ return ((VersionChange)other).version.equals(this.version);
+ }
+
+ @Override
+ public String toString() {
+ return "version change to " + version;
+ }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
new file mode 100644
index 00000000000..75e0f82cdcf
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -0,0 +1,50 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.Zone;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * A deployment of an application in a particular zone.
+ *
+ * @author bratseth
+ */
+public class Deployment {
+
+ private final Zone zone;
+ private final ApplicationRevision revision;
+ private final Version version;
+ private final Instant deployTime;
+
+ public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) {
+ Objects.requireNonNull(zone, "zone cannot be null");
+ Objects.requireNonNull(revision, "revision cannot be null");
+ Objects.requireNonNull(version, "version cannot be null");
+ Objects.requireNonNull(deployTime, "deployTime cannot be null");
+ this.zone = zone;
+ this.revision = revision;
+ this.version = version;
+ this.deployTime = deployTime;
+ }
+
+ /** Returns the zone this was deployed to */
+ public Zone zone() { return zone; }
+
+ /** Returns the revision of the application which was deployed */
+ public ApplicationRevision revision() { return revision; }
+
+ /** Returns the Vespa version which was deployed */
+ public Version version() { return version; }
+
+ /** Returns the time this was deployed */
+ public Instant at() { return deployTime; }
+
+ @Override
+ public String toString() {
+ return "deployment to " + zone + " of " + revision + " on version " + version + " at " + deployTime;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
new file mode 100644
index 00000000000..d9256f94086
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -0,0 +1,333 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.Controller;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Information about which deployment jobs an application should run and their current status.
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class DeploymentJobs {
+
+ private final Optional<Long> projectId;
+ private final ImmutableMap<JobType, JobStatus> status;
+ private final Optional<String> jiraIssueId;
+ private final boolean selfTriggering; // TODO: Remove this when no projects are self-triggering.
+
+ /** Creates an empty set of deployment jobs */
+ public DeploymentJobs(long projectId) {
+ this(Optional.of(projectId), ImmutableMap.of(), Optional.empty(),true);
+ }
+
+ public DeploymentJobs(Optional<Long> projectId, Collection<JobStatus> jobStatusEntries, Optional<String> jiraIssueId, boolean selfTriggering) {
+ this(projectId, asMap(jobStatusEntries), jiraIssueId, selfTriggering);
+ }
+
+ private DeploymentJobs(Optional<Long> projectId, Map<JobType, JobStatus> status, Optional<String> jiraIssueId, boolean selfTriggering) {
+ Objects.requireNonNull(projectId, "projectId cannot be null");
+ Objects.requireNonNull(status, "status cannot be null");
+ Objects.requireNonNull(jiraIssueId, "jiraIssueId cannot be null");
+ this.projectId = projectId;
+ this.status = ImmutableMap.copyOf(status);
+ this.jiraIssueId = jiraIssueId;
+ this.selfTriggering = selfTriggering;
+ }
+
+ private static Map<JobType, JobStatus> asMap(Collection<JobStatus> jobStatusEntries) {
+ ImmutableMap.Builder<JobType, JobStatus> b = new ImmutableMap.Builder<>();
+ for (JobStatus jobStatusEntry : jobStatusEntries)
+ b.put(jobStatusEntry.type(), jobStatusEntry);
+ return b.build();
+ }
+
+ /** Return a new instance with the given completion */
+ public DeploymentJobs withCompletion(JobReport report, Instant notificationTime, Controller controller) {
+ Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status);
+ status.compute(report.jobType(), (type, job) -> {
+ if (job == null) job = JobStatus.initial(report.jobType());
+ return job.withCompletion(report.jobError(), notificationTime, controller);
+ });
+ return new DeploymentJobs(Optional.of(report.projectId()), status, jiraIssueId, report.selfTriggering());
+ }
+
+ public DeploymentJobs withTriggering(DeploymentJobs.JobType jobType,
+ Version version,
+ Optional<ApplicationRevision> revision,
+ Instant triggerTime) {
+ Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status);
+ status.compute(jobType, (type, job) -> {
+ if (job == null) job = JobStatus.initial(jobType);
+ return job.withTriggering(version, revision, triggerTime);
+ });
+ return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering);
+ }
+
+ public DeploymentJobs withProjectId(long projectId) {
+ return new DeploymentJobs(Optional.of(projectId), status, jiraIssueId, selfTriggering);
+ }
+
+ public DeploymentJobs withJiraIssueId(Optional<String> jiraIssueId) {
+ return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering);
+ }
+
+ public DeploymentJobs without(JobType job) {
+ Map<JobType, JobStatus> status = new HashMap<>(this.status);
+ status.remove(job);
+ return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering);
+ }
+
+ public DeploymentJobs asSelfTriggering(boolean selfTriggering) {
+ return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering);
+ }
+
+ /** Returns an immutable map of the status entries in this */
+ public Map<JobType, JobStatus> jobStatus() { return status; }
+
+ /** Returns whether this application's deployment jobs trigger each other, and should be left alone, or not. */
+ public boolean isSelfTriggering() { return selfTriggering; }
+
+ /** Returns whether this has some job status which is not a success */
+ public boolean hasFailures() {
+ return status.values().stream().anyMatch(jobStatus -> ! jobStatus.isSuccess());
+ }
+
+ /** Returns whether any job is currently in progress */
+ public boolean inProgress() {
+ return status.values().stream().anyMatch(JobStatus::inProgress);
+ }
+
+ /** Returns whether any job is failing for the given change */
+ public boolean failingOn(Change change) {
+ return status.values().stream().anyMatch(jobStatus -> !jobStatus.isSuccess() && jobStatus.lastCompletedFor(change));
+ }
+
+ /** Returns whether change can be deployed to the given environment */
+ public boolean isDeployableTo(Environment environment, Optional<Change> change) {
+ if (environment == null || !change.isPresent()) {
+ return true;
+ }
+ if (environment == Environment.staging) {
+ return isSuccessful(JobType.systemTest, change.get());
+ } else if (environment == Environment.prod) {
+ return isSuccessful(JobType.stagingTest, change.get());
+ }
+ return true; // other environments do not have any preconditions
+ }
+
+ /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */
+ public Instant failingSince() {
+ Instant failingSince = null;
+ for (JobStatus jobStatus : jobStatus().values()) {
+ if (jobStatus.isSuccess()) continue;
+ if (failingSince == null || failingSince.isAfter(jobStatus.firstFailing().get().at()))
+ failingSince = jobStatus.firstFailing().get().at();
+ }
+ return failingSince;
+ }
+
+ /** Returns the time at which the oldest running job started */
+ public Optional<Instant> runningSince() {
+ return jobStatus().values().stream()
+ .filter(JobStatus::inProgress)
+ .sorted(Comparator.comparing(jobStatus -> jobStatus.lastTriggered().get().at()))
+ .map(jobStatus -> jobStatus.lastTriggered().get().at())
+ .findFirst();
+ }
+
+ /**
+ * Returns the id of the Screwdriver project running these deployment jobs
+ * - or empty when this is not known or does not exist.
+ * It is not known until the jobs have run once and reported back to the controller.
+ */
+ public Optional<Long> projectId() { return projectId; }
+
+ public Optional<String> jiraIssueId() { return jiraIssueId; }
+
+ private boolean isSuccessful(JobType jobType, Change change) {
+ return Optional.ofNullable(jobStatus().get(jobType))
+ .filter(JobStatus::isSuccess)
+ .filter(status -> status.lastCompletedFor(change))
+ .isPresent();
+ }
+
+ /** Job types that exist in the build system */
+ public enum JobType {
+
+ component("component"),
+ systemTest("system-test", zone(SystemName.cd, "test", "cd-us-central-1"), zone("test", "us-east-1")),
+ stagingTest("staging-test", zone(SystemName.cd, "staging", "cd-us-central-1"), zone("staging", "us-east-3")),
+ productionCorpUsEast1("production-corp-us-east-1", zone("prod", "corp-us-east-1")),
+ productionUsEast3("production-us-east-3", zone("prod", "us-east-3")),
+ productionUsWest1("production-us-west-1", zone("prod", "us-west-1")),
+ productionUsCentral1("production-us-central-1", zone("prod", "us-central-1")),
+ productionApNortheast1("production-ap-northeast-1", zone("prod", "ap-northeast-1")),
+ productionApNortheast2("production-ap-northeast-2", zone("prod", "ap-northeast-2")),
+ productionApSoutheast1("production-ap-southeast-1", zone("prod", "ap-southeast-1")),
+ productionEuWest1("production-eu-west-1", zone("prod", "eu-west-1")),
+ productionCdUsCentral1("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")),
+ productionCdUsCentral2("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2"));
+
+ private final String id;
+ private final Map<SystemName, Zone> zones;
+
+ JobType(String id, Zone... zone) {
+ this.id = id;
+ Map<SystemName, Zone> zones = new HashMap<>();
+ for (Zone z : zone) {
+ if (zones.containsKey(z.system())) {
+ throw new IllegalArgumentException("A job can only map to a single zone per system");
+ }
+ zones.put(z.system(), z);
+ }
+ this.zones = Collections.unmodifiableMap(zones);
+ }
+
+ public String id() { return id; }
+
+ /** Returns the zone for this job in the given system, or empty if this job does not have a zone */
+ public Optional<Zone> zone(SystemName system) {
+ return Optional.ofNullable(zones.get(system));
+ }
+
+ /** Returns whether this is a production job */
+ public boolean isProduction() { return environment() == Environment.prod; }
+
+ /** Returns the environment of this job type, or null if it does not have an environment */
+ public Environment environment() {
+ switch (this) {
+ case component: return null;
+ case systemTest: return Environment.test;
+ case stagingTest: return Environment.staging;
+ default: return Environment.prod;
+ }
+ }
+
+ /** Returns the region of this job type, or null if it does not have a region */
+ public RegionName region(SystemName system) {
+ return zone(system).map(Zone::region).orElse(null);
+ }
+
+ public static JobType fromId(String id) {
+ switch (id) {
+ case "component" : return component;
+ case "system-test" : return systemTest;
+ case "staging-test" : return stagingTest;
+ case "production-corp-us-east-1" : return productionCorpUsEast1;
+ case "production-us-east-3" : return productionUsEast3;
+ case "production-us-west-1" : return productionUsWest1;
+ case "production-us-central-1" : return productionUsCentral1;
+ case "production-ap-northeast-1" : return productionApNortheast1;
+ case "production-ap-northeast-2" : return productionApNortheast2;
+ case "production-ap-southeast-1" : return productionApSoutheast1;
+ case "production-eu-west-1" : return productionEuWest1;
+ case "production-cd-us-central-1" : return productionCdUsCentral1;
+ case "production-cd-us-central-2" : return productionCdUsCentral2;
+ default : throw new IllegalArgumentException("Unknown job id '" + id + "'");
+ }
+ }
+
+ /** Returns the job type for the given zone, or null if none */
+ public static JobType from(SystemName system, com.yahoo.config.provision.Zone zone) {
+ for (JobType job : values()) {
+ Optional<com.yahoo.config.provision.Zone> jobZone = job.zone(system);
+ if (jobZone.isPresent() && jobZone.get().equals(zone))
+ return job;
+ }
+ return null;
+ }
+
+ /** Returns the job job type for the given environment and region or null if none */
+ public static JobType from(SystemName system, Environment environment, RegionName region) {
+ switch (environment) {
+ case test: return systemTest;
+ case staging: return stagingTest;
+ }
+ return from(system, new com.yahoo.config.provision.Zone(environment, region));
+ }
+
+ /** Returns the trigger order to use according to deployment spec */
+ public static List<JobType> triggerOrder(SystemName system, DeploymentSpec deploymentSpec) {
+ return deploymentSpec.zones().stream()
+ .map(declaredZone -> JobType.from(system, declaredZone.environment(),
+ declaredZone.region().orElse(null)))
+ .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+ }
+
+ private static Zone zone(SystemName system, String environment, String region) {
+ return new Zone(system, Environment.from(environment), RegionName.from(region));
+ }
+
+ private static Zone zone(String environment, String region) {
+ return new Zone(Environment.from(environment), RegionName.from(region));
+ }
+ }
+
+ /** A job report. This class is immutable. */
+ public static class JobReport {
+
+ private final ApplicationId applicationId;
+ private final JobType jobType;
+ private final long projectId;
+ private final long buildNumber;
+ private final Optional<JobError> jobError;
+ private final boolean selfTriggering;
+ private final boolean gitChanges;
+
+ public JobReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber, Optional<JobError> jobError, boolean selfTriggering, boolean gitChanges) {
+ Objects.requireNonNull(applicationId, "ApplicationId can not be null.");
+ Objects.requireNonNull(jobType, "JobType can not be null.");
+
+ this.applicationId = applicationId;
+ this.projectId = projectId;
+ this.jobType = jobType;
+ this.buildNumber = buildNumber;
+ this.jobError = jobError;
+ this.selfTriggering = selfTriggering;
+ this.gitChanges = gitChanges;
+ }
+
+ public ApplicationId applicationId() { return applicationId; }
+ public JobType jobType() { return jobType; }
+ public long projectId() { return projectId; }
+ public long buildNumber() { return buildNumber; }
+ public boolean success() { return !jobError.isPresent(); }
+ public Optional<JobError> jobError() { return jobError; }
+ public boolean selfTriggering() { return selfTriggering; }
+ public boolean gitChanges() { return gitChanges; }
+
+ }
+
+ public enum JobError {
+ unknown,
+ outOfCapacity;
+
+ public static Optional<JobError> from(boolean success) {
+ return Optional.of(success)
+ .filter(b -> !b)
+ .map(ignored -> unknown);
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
new file mode 100644
index 00000000000..a30998d8517
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
@@ -0,0 +1,209 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.Controller;
+
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * The last known build status of a particular deployment job for a particular application.
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class JobStatus {
+
+ private final DeploymentJobs.JobType type;
+
+ private final Optional<JobRun> lastTriggered;
+ private final Optional<JobRun> lastCompleted;
+ private final Optional<JobRun> firstFailing;
+ private final Optional<JobRun> lastSuccess;
+
+ private final Optional<DeploymentJobs.JobError> jobError;
+
+ /**
+ * Used by the persistence layer (only) to create a complete JobStatus instance.
+ * Other creation should be by using initial- and with- methods.
+ */
+ public JobStatus(DeploymentJobs.JobType type, Optional<DeploymentJobs.JobError> jobError,
+ Optional<JobRun> lastTriggered, Optional<JobRun> lastCompleted,
+ Optional<JobRun> firstFailing, Optional<JobRun> lastSuccess) {
+ Objects.requireNonNull(type, "jobType cannot be null");
+ Objects.requireNonNull(jobError, "jobError cannot be null");
+ Objects.requireNonNull(lastTriggered, "lastTriggered cannot be null");
+ Objects.requireNonNull(lastCompleted, "lastCompleted cannot be null");
+ Objects.requireNonNull(firstFailing, "firstFailing cannot be null");
+ Objects.requireNonNull(lastSuccess, "lastSuccess cannot be null");
+
+ this.type = type;
+ this.jobError = jobError;
+ this.lastTriggered = lastTriggered;
+ this.lastCompleted = lastCompleted;
+ this.firstFailing = firstFailing;
+ this.lastSuccess = lastSuccess;
+ }
+
+ /** Returns an empty job status */
+ public static JobStatus initial(DeploymentJobs.JobType type) {
+ return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
+ }
+
+ public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, Instant triggerTime) {
+ return new JobStatus(type, jobError, Optional.of(new JobRun(version, revision, triggerTime)),
+ lastCompleted, firstFailing, lastSuccess);
+ }
+
+ public JobStatus withCompletion(Optional<DeploymentJobs.JobError> jobError, Instant completionTime, Controller controller) {
+ Version version;
+ Optional<ApplicationRevision> revision;
+ if (type == DeploymentJobs.JobType.component) { // not triggered by us
+ version = controller.systemVersion();
+ revision = Optional.empty();
+ }
+ else if (! lastTriggered.isPresent()) {
+ throw new IllegalStateException("Got notified about completion of " + this +
+ ", but that has not been triggered nor deployed");
+
+ }
+ else {
+ version = lastTriggered.get().version();
+ revision = lastTriggered.get().revision();
+ }
+
+ JobRun thisCompletion = new JobRun(version, revision, completionTime);
+
+ Optional<JobRun> firstFailing = this.firstFailing;
+ if (jobError.isPresent() && ! this.firstFailing.isPresent())
+ firstFailing = Optional.of(thisCompletion);
+
+ Optional<JobRun> lastSuccess = this.lastSuccess;
+ if ( ! jobError.isPresent()) {
+ lastSuccess = Optional.of(thisCompletion);
+ firstFailing = Optional.empty();
+ }
+
+ return new JobStatus(type, jobError, lastTriggered, Optional.of(thisCompletion), firstFailing, lastSuccess);
+ }
+
+ public DeploymentJobs.JobType type() { return type; }
+
+ /** Returns true unless this job last completed with a failure */
+ public boolean isSuccess() { return ! jobError.isPresent(); }
+
+ /** The error of the last completion, or empty if the last run succeeded */
+ public Optional<DeploymentJobs.JobError> jobError() { return jobError; }
+
+ /** Returns true if job is in progress */
+ public boolean inProgress() {
+ if (!lastTriggered().isPresent()) {
+ return false;
+ }
+ if (!lastCompleted().isPresent()) {
+ return true;
+ }
+ return lastTriggered().get().at().isAfter(lastCompleted().get().at());
+ }
+
+ /**
+ * Returns the last triggering of this job, or empty if the controller has never triggered it
+ * and not seen a deployment for it
+ */
+ public Optional<JobRun> lastTriggered() { return lastTriggered; }
+
+ /** Returns the last completion of this job (whether failing or succeeding), or empty if it never completed */
+ public Optional<JobRun> lastCompleted() { return lastCompleted; }
+
+ /** Returns the run when this started failing, or empty if it is not currently failing */
+ public Optional<JobRun> firstFailing() { return firstFailing; }
+
+ /** Returns the run when this last succeeded, or empty if it has never succeeded */
+ public Optional<JobRun> lastSuccess() { return lastSuccess; }
+
+ /** Returns whether the job last completed for the given change */
+ public boolean lastCompletedFor(Change change) {
+ if (change instanceof Change.ApplicationChange) {
+ Change.ApplicationChange applicationChange = (Change.ApplicationChange) change;
+ return lastCompleted().isPresent() && lastCompleted().get().revision().equals(applicationChange.revision());
+ } else if (change instanceof Change.VersionChange) {
+ Change.VersionChange versionChange = (Change.VersionChange) change;
+ return lastCompleted().isPresent() && lastCompleted().get().version().equals(versionChange.version());
+ }
+ throw new IllegalArgumentException("Unexpected change: " + change.getClass());
+ }
+
+ @Override
+ public String toString() {
+ return "job status of " + type + "[ " +
+ "last triggered: " + lastTriggered.map(JobRun::toString).orElse("(never)") +
+ ", last completed: " + lastCompleted.map(JobRun::toString).orElse("(never)") +
+ ", first failing: " + firstFailing.map(JobRun::toString).orElse("(not failing)") +
+ ", lastSuccess: " + lastSuccess.map(JobRun::toString).orElse("(never)") + "]";
+ }
+
+ @Override
+ public int hashCode() { return Objects.hash(type, jobError, lastTriggered, lastCompleted, firstFailing, lastSuccess); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) return true;
+ if ( ! ( o instanceof JobStatus)) return false;
+ JobStatus other = (JobStatus)o;
+ return Objects.equals(type, other.type) &&
+ Objects.equals(jobError, other.jobError) &&
+ Objects.equals(lastTriggered, other.lastTriggered) &&
+ Objects.equals(lastCompleted, other.lastCompleted) &&
+ Objects.equals(firstFailing, other.firstFailing) &&
+ Objects.equals(lastSuccess, other.lastSuccess);
+ }
+
+ /** Information about a particular triggering or completion of a run of a job. This is immutable. */
+ public static class JobRun {
+
+ private final Version version;
+ private final Optional<ApplicationRevision> revision;
+ private final Instant at;
+
+ public JobRun(Version version, Optional<ApplicationRevision> revision, Instant at) {
+ Objects.requireNonNull(version, "version cannot be null");
+ Objects.requireNonNull(revision, "revision cannot be null");
+ Objects.requireNonNull(at, "at cannot be null");
+ this.version = version;
+ this.revision = revision;
+ this.at = at;
+ }
+
+ /** The Vespa version used on this run */
+ public Version version() { return version; }
+
+ /** The application revision used for this run, or empty when not known */
+ public Optional<ApplicationRevision> revision() { return revision; }
+
+ /** The time if this triggering or completion */
+ public Instant at() { return at; }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(version ,revision, at);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) return true;
+ if ( ! (o instanceof JobRun)) return false;
+ JobRun other = (JobRun)o;
+ if ( ! Objects.equals(other.version, this.version)) return false;
+ if ( ! Objects.equals(this.revision, other.revision)) return false;
+ if ( ! Objects.equals(this.at, other.at)) return false;
+ return true;
+ }
+
+ @Override
+ public String toString() { return "job run of version " + version + " " + revision + " at " + at; }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java
new file mode 100644
index 00000000000..9c10e0dc153
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SourceRevision.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import java.util.Objects;
+
+/**
+ * A revision in a source repository
+ *
+ * @author bratseth
+ */
+public class SourceRevision {
+
+ private final String repository;
+ private final String branch;
+ private final String commit;
+
+ public SourceRevision(String repository, String branch, String commit) {
+ Objects.requireNonNull(repository, "repository cannot be null");
+ Objects.requireNonNull(branch, "branch cannot be null");
+ Objects.requireNonNull(commit, "commit cannot be null");
+ this.repository = repository;
+ this.branch = branch;
+ this.commit = commit;
+ }
+
+ public String repository() { return repository; }
+ public String branch() { return branch; }
+ public String commit() { return commit; }
+
+ @Override
+ public int hashCode() { return Objects.hash(repository, branch, commit); }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) return true;
+ if ( ! (o instanceof SourceRevision)) return false;
+
+ SourceRevision other = (SourceRevision)o;
+ return this.repository.equals(other.repository) &&
+ this.branch.equals(other.branch) &&
+ this.commit.equals(other.commit);
+ }
+
+ @Override
+ public String toString() { return "source revision of repository '" + repository +
+ "', branch '" + branch + "' with commit '" + commit + "'"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java
new file mode 100644
index 00000000000..69c846f2562
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ZipStreamReader.java
@@ -0,0 +1,63 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.google.common.collect.ImmutableList;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * @author bratseth
+ */
+public class ZipStreamReader {
+
+ private final ImmutableList<ZipEntryWithContent> entries;
+
+ public ZipStreamReader(InputStream input) {
+ try (ZipInputStream zipInput = new ZipInputStream(input)) {
+ ImmutableList.Builder<ZipEntryWithContent> builder = new ImmutableList.Builder<>();
+ ZipEntry zipEntry;
+ while (null != (zipEntry = zipInput.getNextEntry()))
+ builder.add(new ZipEntryWithContent(zipEntry, readContent(zipInput)));
+ entries = builder.build();
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("IO error reading zip content", e);
+ }
+ }
+
+ private byte[] readContent(ZipInputStream zipInput) {
+ try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) {
+ byte[] buffer = new byte[2048];
+ int read;
+ while ( -1 != (read = zipInput.read(buffer)))
+ bis.write(buffer, 0, read);
+ return bis.toByteArray();
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("Failed reading from zipped content", e);
+ }
+ }
+
+ public List<ZipEntryWithContent> entries() { return entries; }
+
+ public static class ZipEntryWithContent {
+
+ private final ZipEntry zipEntry;
+ private final byte[] content;
+
+ public ZipEntryWithContent(ZipEntry zipEntry, byte[] content) {
+ this.zipEntry = zipEntry;
+ this.content = content;
+ }
+
+ public ZipEntry zipEntry() { return zipEntry; }
+ public byte[] content() { return content; }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java
new file mode 100644
index 00000000000..4dbce299b5d
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Core application model
+ *
+ * @author bratseth
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java
new file mode 100644
index 00000000000..df80fafd388
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Lock.java
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.concurrent;
+
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * An acquired lock which is released on close
+ *
+ * @author bratseth
+ */
+public final class Lock implements AutoCloseable {
+
+ private final ReentrantLock wrappedLock;
+
+ Lock(ReentrantLock wrappedLock) {
+ this.wrappedLock = wrappedLock;
+ }
+
+ /** Releases this lock */
+ public void close() {
+ wrappedLock.unlock();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java
new file mode 100644
index 00000000000..6168812203a
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/Locks.java
@@ -0,0 +1,55 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.concurrent;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Holds a map of locks indexed on keys of a given type.
+ * This is suitable in cases where exclusive access should be granted to any one of a set of keyed objects and
+ * there is a finite collection of keyed objects.
+ *
+ * The returned locks are reentrant (i.e the owning thread may call lock multiple times) and auto-closable.
+ *
+ * Typical use is
+ * <code>
+ * try (Lock lock = locks.lock(id)) {
+ * exclusive use of the object with key id
+ * }
+ * </code>
+ *
+ * @author bratseth
+ */
+public class Locks<TYPE> {
+
+ private final Map<TYPE, ReentrantLock> locks = new ConcurrentHashMap<>();
+
+ private final long timeoutMs;
+
+ public Locks(int timeout, TimeUnit timeoutUnit) {
+ timeoutMs = timeoutUnit.toMillis(timeout);
+ }
+
+ /**
+ * Locks key. This will block until the key is acquired.
+ * Users of this <b>must</b> close any lock acquired.
+ *
+ * @param key the key to lock
+ * @return the acquired lock
+ * @throws TimeoutException if the lock could not be acquired within the timeout
+ */
+ public Lock lock(TYPE key) {
+ try {
+ ReentrantLock lock = locks.computeIfAbsent(key, k -> new ReentrantLock(true));
+ boolean acquired = lock.tryLock(timeoutMs, TimeUnit.MILLISECONDS);
+ if ( ! acquired)
+ throw new TimeoutException("Timed out waiting for the lock to " + key);
+ return new Lock(lock);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted while waiting for lock of " + key);
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java
new file mode 100644
index 00000000000..260761fa6ac
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java
@@ -0,0 +1,15 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.concurrent;
+
+/**
+ * Throws on timeout
+ *
+ * @author bratseth
+ */
+public class TimeoutException extends RuntimeException {
+
+ public TimeoutException(String message) {
+ super(message);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java
new file mode 100644
index 00000000000..15b3ef7fb83
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/BuildSystem.java
@@ -0,0 +1,34 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+
+import java.util.List;
+
+/**
+ * @author jvenstad
+ * @author mpolden
+ */
+public interface BuildSystem {
+
+ /**
+ * Add a job for the given application to the build system
+ *
+ * @param application the application owning the job
+ * @param jobType the job type to be queued
+ * @param first whether the job should be added to the front of the queue
+ */
+ void addJob(ApplicationId application, JobType jobType, boolean first);
+
+ /** Remove and return a list of jobs which should be run now */
+ List<BuildJob> takeJobsToRun();
+
+ /** Get a list of all jobs currently waiting to run */
+ List<BuildJob> jobs();
+
+ /** Removes all queued jobs for the given application */
+ void removeJobs(ApplicationId applicationId);
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
new file mode 100644
index 00000000000..2bc219dde62
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -0,0 +1,368 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+/**
+ * Responsible for scheduling deployment jobs in a build system and keeping
+ * Application.deploying() in sync with what is scheduled.
+ *
+ * This class is multithread safe.
+ *
+ * @author bratseth
+ */
+public class DeploymentTrigger {
+
+ private final static Logger log = Logger.getLogger(DeploymentTrigger.class.getName());
+
+ private final Controller controller;
+ private final Clock clock;
+ private final BuildSystem buildSystem;
+
+ public DeploymentTrigger(Controller controller, CuratorDb curator, Clock clock) {
+ Objects.requireNonNull(controller,"controller cannot be null");
+ Objects.requireNonNull(clock,"clock cannot be null");
+ this.controller = controller;
+ this.clock = clock;
+ this.buildSystem = new PolledBuildSystem(controller, curator);
+ }
+
+ //--- Start of methods which triggers deployment jobs -------------------------
+
+ /**
+ * Called each time a job completes (successfully or not) to cause triggering of one or more follow-up jobs
+ * (which may possibly the same job once over).
+ *
+ * @param report information about the job that just completed
+ */
+ public void triggerFromCompletion(JobReport report) {
+ try (Lock lock = applications().lock(report.applicationId())) {
+ Application application = applications().require(report.applicationId());
+ application = application.withJobCompletion(report, clock.instant(), controller);
+
+ // Handle successful first and last job
+ if (isFirstJob(report.jobType()) && report.success()) { // the first job tells us that a change occurred
+ if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) { // postpone until the current deployment is done
+ applications().store(application.withOutstandingChange(true), lock);
+ return;
+ }
+ else { // start a new change deployment
+ application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown()));
+ }
+ } else if (isLastJob(report.jobType(), application) && report.success()) {
+ application = application.withDeploying(Optional.empty());
+ }
+
+ // Trigger next
+ if (report.success())
+ application = trigger(nextAfter(report.jobType(), application), application, report.jobType() + " completed successfully", lock);
+ else if (isCapacityConstrained(report.jobType()) && shouldRetryOnOutOfCapacity(application, report.jobType()))
+ application = trigger(report.jobType(), application, true, "Retrying due to out of capacity", lock);
+ else if (shouldRetryNow(application))
+ application = trigger(report.jobType(), application, "Retrying as job just started failing", lock);
+
+ applications().store(application, lock);
+ }
+ }
+
+ /**
+ * Called periodically to cause triggering of jobs in the background
+ */
+ public void triggerFailing(ApplicationId applicationId) {
+ try (Lock lock = applications().lock(applicationId)) {
+ Application application = applications().require(applicationId);
+ if (shouldRetryFromBeginning(application)) {
+ // failed for a long time: Discard existing change and restart from the component job
+ application = application.withDeploying(Optional.empty());
+ application = trigger(JobType.component, application, "Retrying failing deployment from beginning", lock);
+ applications().store(application, lock);
+ } else {
+ // retry the failed job (with backoff)
+ for (JobType jobType : JobType.triggerOrder(controller.system(), application.deploymentSpec())) { // retry the *first* failing job
+ JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
+ if (isFailing(jobStatus)) {
+ if (shouldRetryNow(jobStatus)) {
+ application = trigger(jobType, application, "Retrying failing job", lock);
+ applications().store(application, lock);
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ /** Triggers jobs that have been delayed according to deployment spec */
+ public void triggerDelayed() {
+ for (Application application : applications().asList()) {
+ if ( ! application.deploying().isPresent() ) continue;
+ if (application.deploymentJobs().hasFailures()) continue;
+ if (application.deploymentJobs().inProgress()) continue;
+
+ Optional<JobStatus> lastSuccessfulJob = application.deploymentJobs().jobStatus().values()
+ .stream()
+ .filter(j -> j.lastSuccess().isPresent())
+ .sorted(Comparator.<JobStatus, Instant>comparing(j -> j.lastSuccess().get().at()).reversed())
+ .findFirst();
+ if ( ! lastSuccessfulJob.isPresent() ) continue;
+
+ // Trigger next
+ try (Lock lock = applications().lock(application.id())) {
+ application = applications().require(application.id());
+ application = trigger(nextAfter(lastSuccessfulJob.get().type(), application), application,
+ "Delayed by deployment spec", lock);
+ applications().store(application, lock);
+ }
+ }
+ }
+
+ /**
+ * Triggers a change of this application
+ *
+ * @param applicationId the application to trigger
+ * @throws IllegalArgumentException if this application already have an ongoing change
+ */
+ public void triggerChange(ApplicationId applicationId, Change change) {
+ try (Lock lock = applications().lock(applicationId)) {
+ Application application = applications().require(applicationId);
+ if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures())
+ throw new IllegalArgumentException("Could not upgrade " + application + ": A change is already in progress");
+ application = application.withDeploying(Optional.of(change));
+ if (change instanceof Change.ApplicationChange)
+ application = application.withOutstandingChange(false);
+ application = trigger(JobType.systemTest, application, "Deploying change", lock);
+ applications().store(application, lock);
+ }
+ }
+
+ /**
+ * Cancels any ongoing upgrade of the given application
+ *
+ * @param applicationId the application to trigger
+ */
+ public void cancelChange(ApplicationId applicationId) {
+ try (Lock lock = applications().lock(applicationId)) {
+ Application application = applications().require(applicationId);
+ buildSystem.removeJobs(application.id());
+ application = application.withDeploying(Optional.empty());
+ applications().store(application, lock);
+ }
+ }
+
+ //--- End of methods which triggers deployment jobs ----------------------------
+
+ private ApplicationController applications() { return controller.applications(); }
+
+ /** Returns the next job to trigger after this job, or null if none should be triggered */
+ private JobType nextAfter(JobType jobType, Application application) {
+ // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a
+ // new application with no previous deployments)
+ if (jobType == JobType.component) {
+ return JobType.systemTest;
+ }
+
+ // At this point we've at least deployed to system test, so deployment spec should be available
+ List<DeploymentSpec.DeclaredZone> zones = application.deploymentSpec().zones();
+ Optional<DeploymentSpec.DeclaredZone> zoneForJob = zoneForJob(application, jobType);
+ if (!zoneForJob.isPresent()) {
+ return null;
+ }
+ int zoneIndex = application.deploymentSpec().zones().indexOf(zoneForJob.get());
+
+ // This is last zone
+ if (zoneIndex == zones.size() - 1) {
+ return null;
+ }
+
+ // Skip next job if delay has not passed yet
+ Duration delay = delayAfter(application, zoneForJob.get());
+ Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType))
+ .flatMap(JobStatus::lastSuccess)
+ .map(JobStatus.JobRun::at);
+ if (lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant())) {
+ log.info(String.format("Delaying next job after %s of %s by %s", jobType, application, delay));
+ return null;
+ }
+
+ DeploymentSpec.DeclaredZone nextZone = application.deploymentSpec().zones().get(zoneIndex + 1);
+ return JobType.from(controller.system(), nextZone.environment(), nextZone.region().orElse(null));
+ }
+
+ private Duration delayAfter(Application application, DeploymentSpec.DeclaredZone zone) {
+ int stepIndex = application.deploymentSpec().steps().indexOf(zone);
+ if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) {
+ return Duration.ZERO;
+ }
+ Duration totalDelay = Duration.ZERO;
+ List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps()
+ .subList(stepIndex + 1, application.deploymentSpec().steps().size());
+ for (DeploymentSpec.Step step : remainingSteps) {
+ if (!(step instanceof DeploymentSpec.Delay)) {
+ break;
+ }
+ totalDelay = totalDelay.plus(((DeploymentSpec.Delay) step).duration());
+ }
+ return totalDelay;
+ }
+
+ private Optional<DeploymentSpec.DeclaredZone> zoneForJob(Application application, JobType jobType) {
+ return application.deploymentSpec()
+ .zones()
+ .stream()
+ .filter(z -> {
+ if (jobType.isProduction()) {
+ return z.matches(jobType.environment(),
+ Optional.ofNullable(jobType.region(controller.system())));
+ } else {
+ // Ignore region for test environments as it's not specified in deployment spec
+ return z.environment() == jobType.environment();
+ }
+ })
+ .findFirst();
+ }
+
+ private boolean isFirstJob(JobType jobType) {
+ return jobType == JobType.component;
+ }
+
+ private boolean isLastJob(JobType jobType, Application application) {
+ List<JobType> triggerOrder = JobType.triggerOrder(controller.system(), application.deploymentSpec());
+ return triggerOrder.isEmpty() || jobType.equals(triggerOrder.get(triggerOrder.size() - 1));
+ }
+
+ private boolean isFailing(JobStatus jobStatusOrNull) {
+ return jobStatusOrNull != null && !jobStatusOrNull.isSuccess();
+ }
+
+ private boolean isCapacityConstrained(JobType jobType) {
+ return jobType == JobType.stagingTest || jobType == JobType.systemTest;
+ }
+
+ private boolean shouldRetryFromBeginning(Application application) {
+ Instant eightHoursAgo = clock.instant().minus(Duration.ofHours(8));
+ Instant failingSince = application.deploymentJobs().failingSince();
+ if (failingSince != null && failingSince.isAfter(eightHoursAgo)) return false;
+
+ JobStatus componentJobStatus = application.deploymentJobs().jobStatus().get(JobType.component);
+ if (componentJobStatus == null) return true;
+ if ( ! componentJobStatus.lastCompleted().isPresent() ) return true;
+ return componentJobStatus.lastCompleted().get().at().isBefore(eightHoursAgo);
+ }
+
+ /** Decide whether the job should be triggered by the periodic trigger */
+ private boolean shouldRetryNow(JobStatus job) {
+ if (job.isSuccess()) return false;
+
+ if ( ! job.lastCompleted().isPresent()) return true; // Retry when we don't hear back
+
+ // Always retry if we haven't tried in 4 hours
+ if (job.lastCompleted().get().at().isBefore(clock.instant().minus(Duration.ofHours(4)))) return true;
+
+ // Wait for 10% of the time since it started failing
+ Duration aTenthOfFailTime = Duration.ofMillis( (clock.millis() - job.firstFailing().get().at().toEpochMilli()) / 10);
+ if (job.lastCompleted().get().at().isBefore(clock.instant().minus(aTenthOfFailTime))) return true;
+
+ return false;
+ }
+
+ /** Retry immediately only if this just started failing. Otherwise retry periodically */
+ private boolean shouldRetryNow(Application application) {
+ return application.deploymentJobs().failingSince().isAfter(clock.instant().minus(Duration.ofSeconds(10)));
+ }
+
+ /** Decide whether to retry due to capacity restrictions */
+ private boolean shouldRetryOnOutOfCapacity(Application application, JobType jobType) {
+ Optional<JobError> outOfCapacityError = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType))
+ .flatMap(JobStatus::jobError)
+ .filter(e -> e.equals(JobError.outOfCapacity));
+
+ if ( ! outOfCapacityError.isPresent()) return false;
+
+ // Retry the job if it failed recently
+ return application.deploymentJobs().jobStatus().get(jobType).firstFailing().get().at()
+ .isAfter(clock.instant().minus(Duration.ofMinutes(15)));
+ }
+
+ /** Decide whether job type should be triggered according to deployment spec */
+ private boolean deploysTo(Application application, JobType jobType) {
+ Optional<Zone> zone = jobType.zone(controller.system());
+ if (zone.isPresent() && jobType.isProduction()) {
+ // Skip triggering of jobs for zones where the application should not be deployed
+ if (!application.deploymentSpec().includes(jobType.environment(), Optional.of(zone.get().region()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Trigger a job for an application
+ *
+ * @param jobType the type of the job to trigger, or null to trigger nothing
+ * @param application the application to trigger the job for
+ * @param first whether to trigger the job before other jobs
+ * @param cause describes why the job is triggered
+ * @return the application in the triggered state, which *must* be stored by the caller
+ */
+ private Application trigger(JobType jobType, Application application, boolean first, String cause, Lock lock) {
+ if (jobType == null) return application; // previous was last job
+
+ // TODO: Remove when we can determine why this occurs
+ if (jobType != JobType.component && !application.deploying().isPresent()) {
+ log.warning(String.format("Want to trigger %s for %s with reason %s, but this application is not " +
+ "currently deploying a change",
+ jobType, application, cause));
+ return application;
+ }
+
+ if (!deploysTo(application, jobType)) {
+ return application;
+ }
+
+ if (!application.deploymentJobs().isDeployableTo(jobType.environment(), application.deploying())) {
+ log.warning(String.format("Want to trigger %s for %s with reason %s, but change is untested", jobType,
+ application, cause));
+ return application;
+ }
+
+ if (application.deploymentJobs().isSelfTriggering()) {
+ log.info("Not triggering " + jobType + " for self-triggering " + application);
+ return application;
+ }
+
+ log.info(String.format("Triggering %s for %s, %s: %s", jobType, application,
+ application.deploying().map(d -> "deploying " + d).orElse("restarted deployment"),
+ cause));
+ buildSystem.addJob(application.id(), jobType, first);
+
+ return application.withJobTriggering(jobType, clock.instant(), controller);
+ }
+
+ private Application trigger(JobType jobType, Application application, String cause, Lock lock) {
+ return trigger(jobType, application, false, cause, lock);
+ }
+
+ public BuildSystem buildSystem() { return buildSystem; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java
new file mode 100644
index 00000000000..41adb4abe6a
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java
@@ -0,0 +1,100 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.List;
+
+/**
+ * Stores a queue for each type of job, and offers jobs from each of these to a periodic
+ * polling mechanism which is responsible for triggering the offered jobs in an external build service.
+ *
+ * @author jvenstad
+ * @author mpolden
+ */
+public class PolledBuildSystem implements BuildSystem {
+
+ private final Controller controller;
+
+ private final CuratorDb curator;
+
+ public PolledBuildSystem(Controller controller, CuratorDb curator) {
+ this.controller = controller;
+ this.curator = curator;
+ }
+
+ @Override
+ public void addJob(ApplicationId application, JobType jobType, boolean first) {
+ try (Lock lock = curator.lockJobQueues()) {
+ Deque<ApplicationId> queue = curator.readJobQueue(jobType);
+ if ( ! queue.contains(application)) {
+ if (first) {
+ queue.addFirst(application);
+ } else {
+ queue.add(application);
+ }
+ }
+ curator.writeJobQueue(jobType, queue);
+ }
+ }
+
+ @Override
+ public List<BuildJob> jobs() {
+ return getJobs(false);
+ }
+
+ @Override
+ public List<BuildJob> takeJobsToRun() {
+ return getJobs(true);
+ }
+
+
+ @Override
+ public void removeJobs(ApplicationId application) {
+ try (Lock lock = curator.lockJobQueues()) {
+ for (JobType jobType : JobType.values()) {
+ Deque<ApplicationId> queue = curator.readJobQueue(jobType);
+ while (queue.remove(application)) {
+ // keep removing until not found
+ }
+ curator.writeJobQueue(jobType, queue);
+ }
+ }
+ }
+
+ private List<BuildJob> getJobs(boolean removeFromQueue) {
+ try (Lock lock = curator.lockJobQueues()) {
+ List<BuildJob> jobsToRun = new ArrayList<>();
+ for (JobType jobType : JobType.values()) {
+ Deque<ApplicationId> queue = curator.readJobQueue(jobType);
+ for (ApplicationId a : queue) {
+ ApplicationId application = removeFromQueue ? queue.poll() : a;
+ jobsToRun.add(new BuildJob(projectIdFor(application), jobType.id()));
+
+ // Return only one job at a time for capacity constrained queues
+ if (removeFromQueue && isCapacityConstrained(jobType)) break;
+ }
+ if (removeFromQueue)
+ curator.writeJobQueue(jobType, queue);
+ }
+ return Collections.unmodifiableList(jobsToRun);
+ }
+ }
+
+ private Long projectIdFor(ApplicationId applicationId) {
+ return controller.applications().require(applicationId).deploymentJobs().projectId().get();
+ }
+
+ private static boolean isCapacityConstrained(JobType jobType) {
+ return jobType == JobType.stagingTest || jobType == JobType.systemTest;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
new file mode 100644
index 00000000000..016ea66cb1a
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -0,0 +1,67 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.Contacts;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues;
+import com.yahoo.vespa.hosted.controller.api.integration.Properties;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
+import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
+
+import java.time.Duration;
+
+/**
+ * Maintenance jobs of the controller.
+ * Each maintenance job is a singleton instance of its implementing class, created and owned by this,
+ * and running its own dedicated thread.
+ *
+ * @author bratseth
+ */
+public class ControllerMaintenance extends AbstractComponent {
+
+ private final JobControl jobControl;
+
+ private final DeploymentExpirer deploymentExpirer;
+ private final DeploymentIssueReporter deploymentIssueReporter;
+ private final MetricsReporter metricsReporter;
+ private final FailureRedeployer failureRedeployer;
+ private final OutstandingChangeDeployer outstandingChangeDeployer;
+ private final VersionStatusUpdater versionStatusUpdater;
+ private final Upgrader upgrader;
+ private final DelayedDeployer delayedDeployer;
+
+ @SuppressWarnings("unused") // instantiated by Dependency Injection
+ public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller,
+ JobControl jobControl, Metric metric, Chef chefClient,
+ Contacts contactsClient, Properties propertiesClient, Issues issuesClient) {
+ Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
+ this.jobControl = jobControl;
+ deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl);
+ deploymentIssueReporter = new DeploymentIssueReporter(controller, contactsClient, propertiesClient,
+ issuesClient, maintenanceInterval, jobControl);
+ metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system());
+ failureRedeployer = new FailureRedeployer(controller, maintenanceInterval, jobControl);
+ outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl);
+ versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl);
+ upgrader = new Upgrader(controller, maintenanceInterval, jobControl);
+ delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl);
+ }
+
+ /** Returns control of the maintenance jobs of this */
+ public JobControl jobControl() { return jobControl; }
+
+ @Override
+ public void deconstruct() {
+ deploymentExpirer.deconstruct();
+ deploymentIssueReporter.deconstruct();
+ metricsReporter.deconstruct();
+ failureRedeployer.deconstruct();
+ outstandingChangeDeployer.deconstruct();
+ versionStatusUpdater.deconstruct();
+ upgrader.deconstruct();
+ delayedDeployer.deconstruct();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java
new file mode 100644
index 00000000000..cb09c41a034
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DelayedDeployer.java
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.Controller;
+
+import java.time.Duration;
+
+/**
+ * Maintenance job which triggers jobs that have been delayed according to the applications deployment spec.
+ *
+ * @author mpolden
+ */
+public class DelayedDeployer extends Maintainer {
+
+ public DelayedDeployer(Controller controller, Duration interval, JobControl jobControl) {
+ super(controller, interval, jobControl);
+ }
+
+ @Override
+ protected void maintain() {
+ controller().applications().deploymentTrigger().triggerDelayed();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
new file mode 100644
index 00000000000..eb44229e790
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
@@ -0,0 +1,66 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.logging.Level;
+
+/**
+ * Expires instances in zones that have configured expiration using TimeToLive.
+ *
+ * @author mortent
+ * @author bratseth
+ */
+public class DeploymentExpirer extends Maintainer {
+
+ private final Clock clock;
+
+ public DeploymentExpirer(Controller controller, Duration interval, JobControl jobControl) {
+ this(controller, interval, Clock.systemUTC(), jobControl);
+ }
+
+ public DeploymentExpirer(Controller controller, Duration interval, Clock clock, JobControl jobControl) {
+ super(controller, interval, jobControl);
+ this.clock = clock;
+ }
+
+ @Override
+ protected void maintain() {
+ for (Application application : controller().applications().asList()) {
+ for (Deployment deployment : application.deployments().values()) {
+ if (deployment.zone().environment().equals(Environment.prod)) continue;
+
+ if (hasExpired(controller().zoneRegistry(), deployment, clock.instant()))
+ deactivate(application, deployment);
+ }
+ }
+ }
+
+ private void deactivate(Application application, Deployment deployment) {
+ try {
+ controller().applications().deactivate(application, deployment, true);
+ }
+ catch (Exception e) {
+ log.log(Level.WARNING, "Could not expire " + deployment + " of " + application, e);
+ }
+ }
+
+ public static boolean hasExpired(ZoneRegistry zoneRegistry, Deployment deployment, Instant now) {
+ return zoneRegistry.getDeploymentTimeToLive(deployment.zone().environment(), deployment.zone().region())
+ .map(duration -> getExpiration(deployment, duration))
+ .map(now::isAfter)
+ .orElse(false);
+ }
+
+ private static Instant getExpiration(Deployment instance, Duration ttl) {
+ return instance.at().plus(ttl);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
new file mode 100644
index 00000000000..90544a8ac30
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
@@ -0,0 +1,234 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.Contacts;
+import com.yahoo.vespa.hosted.controller.api.integration.Contacts.UserContact;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues.Classification;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues.Issue;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo;
+import com.yahoo.vespa.hosted.controller.api.integration.Properties;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.admin;
+import static com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo.Status.done;
+
+/**
+ * Maintenance job which creates Jira issues for tenants when they have jobs which fails continuously
+ * and escalates issues which are not handled.
+ *
+ * @author jvenstad
+ */
+public class DeploymentIssueReporter extends Maintainer {
+
+ static final Duration maxFailureAge = Duration.ofDays(2);
+ static final Duration maxInactivityAge = Duration.ofDays(4);
+ static final String deploymentFailureLabel = "vespaDeploymentFailure";
+ static final Classification vespaOps = new Classification("VESPA", "Services", deploymentFailureLabel);
+ static final UserContact terminalUser = new UserContact("frodelu", "Frode Lundgren", admin);
+
+ private final Contacts contacts;
+ private final Properties properties;
+ private final Issues issues;
+
+ DeploymentIssueReporter(Controller controller, Contacts contacts, Properties properties, Issues issues,
+ Duration maintenanceInterval, JobControl jobControl) {
+ super(controller, maintenanceInterval, jobControl);
+ this.contacts = contacts;
+ this.properties = properties;
+ this.issues = issues;
+ }
+
+ @Override
+ protected void maintain() {
+ maintainDeploymentIssues(controller().applications().asList());
+ escalateInactiveDeploymentIssues(controller().applications().asList());
+ }
+
+ /**
+ * File issues for applications which have failed deployment for longer than @maxFailureAge
+ * and store the issue id for the filed issues. Also, clear the @issueIds of applications
+ * where deployment has not failed for this amount of time.
+ */
+ private void maintainDeploymentIssues(List<Application> applications) {
+ Collection<Application> failingApplications = new ArrayList<>();
+ for (Application application : applications)
+ if (failingSinceBefore(application.deploymentJobs(), controller().clock().instant().minus(maxFailureAge)))
+ failingApplications.add(application);
+ else
+ controller().applications().setJiraIssueId(application.id(), Optional.empty());
+
+ // TODO: Do this when version.confidence is BROKEN instead?
+ if (failingApplications.size() > 0.2 * applications.size()) {
+ fileOrUpdate(manyFailingDeploymentsIssueFrom(failingApplications)); // Problems with Vespa is the most likely cause when so many deployments fail.
+ }
+ else {
+ for (Application application : failingApplications) {
+ Issue deploymentIssue = deploymentIssueFrom(application);
+ Classification applicationOwner = null;
+ try {
+ applicationOwner = jiraClassificationOf(ownerOf(application));
+ fileFor(application, deploymentIssue.with(applicationOwner));
+ }
+ catch (RuntimeException e) { // Catch errors due to inconsistent or missing data in Sherpa, OpsDB, JIRA, and send to ourselves.
+ Pattern componentError = Pattern.compile(".*Component name '.*' is not valid.*", Pattern.DOTALL);
+ if (componentError.matcher(e.getMessage()).matches()) // Several properties seem to list invalid components, in which case we simply ignore this.
+ fileFor(application, deploymentIssue.with(applicationOwner.withComponent(null)));
+ else
+ fileFor(application, deploymentIssue.append(e.getMessage() + "\n\nAddressee:\n" + applicationOwner));
+ }
+ }
+ }
+ }
+
+ /** Returns whether @deploymentJobs has a job which has been failing since before @failureThreshold or not. */
+ private boolean failingSinceBefore(DeploymentJobs deploymentJobs, Instant failureThreshold) {
+ return deploymentJobs.hasFailures() && deploymentJobs.failingSince().isBefore(failureThreshold);
+ }
+
+ private Tenant ownerOf(Application application) {
+ return controller().tenants().tenant(new TenantId(application.id().tenant().value())).get();
+ }
+
+ /** Use the @propertyId of @tenant, if present, to look up JIRA information in OpsDB. */
+ private Classification jiraClassificationOf(Tenant tenant) {
+ Long propertyId = tenant.getPropertyId().map(PropertyId::value).orElseThrow(() ->
+ new NoSuchElementException("No property id is listed for " + tenant));
+
+ Classification classification = properties.classificationFor(propertyId).orElseThrow(() ->
+ new NoSuchElementException("No property was found with id " + propertyId));
+
+ return classification.withLabel(deploymentFailureLabel);
+ }
+
+ /** File @issue for @application, if @application doesn't already have an @Issue associated with it. */
+ private void fileFor(Application application, Issue issue) {
+ Optional<String> ourIssueId = application.deploymentJobs().jiraIssueId()
+ .filter(jiraIssueId -> issues.fetch(jiraIssueId).status() != done);
+
+ if ( ! ourIssueId.isPresent())
+ controller().applications().setJiraIssueId(application.id(), Optional.of(issues.file(issue)));
+ }
+
+ /** File @issue, or update a JIRA issue representing the same issue. */
+ private void fileOrUpdate(Issue issue) {
+ Optional<String> jiraIssueId = issues.fetchSimilarTo(issue)
+ .stream().findFirst().map(Issues.IssueInfo::id);
+
+ if (jiraIssueId.isPresent())
+ issues.update(jiraIssueId.get(), issue.description());
+ else
+ issues.file(issue);
+ }
+
+ /** Escalate JIRA issues for which there has been no activity for a set amount of time. */
+ private void escalateInactiveDeploymentIssues(List<Application> applications) {
+ applications.forEach(application ->
+ application.deploymentJobs().jiraIssueId().ifPresent(jiraIssueId -> {
+ Issues.IssueInfo issueInfo = issues.fetch(jiraIssueId);
+ if (issueInfo.updated().isBefore(controller().clock().instant().minus(maxInactivityAge)))
+ escalateAndComment(issueInfo, application);
+ }));
+ }
+
+ /** Reassign the JIRA issue for @application one step up in the OpsDb escalation chain, and add an explanatory comment to it. */
+ private void escalateAndComment(IssueInfo issueInfo, Application application) {
+ Optional<String> assignee = issueInfo.assignee();
+ if (assignee.isPresent()) {
+ if (assignee.get().equals(terminalUser.username())) return;
+ issues.addWatcher(issueInfo.id(), assignee.get());
+ }
+
+ Long propertyId = ownerOf(application).getPropertyId().get().value();
+
+ UserContact escalationTarget = contacts.escalationTargetFor(propertyId, assignee.orElse("no one"));
+ if (escalationTarget.is(assignee.orElse("no one")))
+ escalationTarget = terminalUser;
+
+ String comment = deploymentIssueEscalationComment(application, propertyId, assignee.orElse("anyone"));
+
+ issues.comment(issueInfo.id(), comment);
+ issues.reassign(issueInfo.id(), escalationTarget.username());
+ }
+
+ Issue deploymentIssueFrom(Application application) {
+ return new Issue(deploymentIssueSummary(application), deploymentIssueDescription(application))
+ .with(vespaOps);
+ }
+
+ Issue manyFailingDeploymentsIssueFrom(Collection<Application> applications) {
+ return new Issue(
+ "More than 20% of Hosted Vespa deployments are failing",
+ applications.stream()
+ .map(application -> "[" + application.id().toShortString() + "|" + toUrl(application.id()) + "]")
+ .collect(Collectors.joining("\n")),
+ vespaOps);
+ }
+
+ // TODO: Use the method of the same name in ApplicationId
+ private static String toShortString(ApplicationId id) {
+ return id.tenant().value() + "." + id.application().value() +
+ ( id.instance().isDefault() ? "" : "." + id.instance().value() );
+ }
+
+ private String toUrl(ApplicationId applicationId) {
+ return controller().zoneRegistry().getDashboardUri().resolve("/apps" +
+ "/tenant/" + applicationId.tenant().value() +
+ "/application/" + applicationId.application().value()).toString();
+ }
+
+ private String toOpsDbUrl(long propertyId) {
+ return contacts.contactsUri(propertyId).toString();
+
+ }
+
+ /** Returns the summary text what will be assigned to a new issue */
+ private static String deploymentIssueSummary(Application application) {
+ return "[" + toShortString(application.id()) + "] Action required: Repair deployment";
+ }
+
+ /** Returns the description text what will be assigned to a new issue */
+ private String deploymentIssueDescription(Application application) {
+ return "Deployment jobs of the Vespa application " +
+ "[" + toShortString(application.id()) + "|" + toUrl(application.id()) + "] have been failing " +
+ "continuously for over 48 hours. This blocks any change to this application from being deployed " +
+ "and will also block global rollout of new Vespa versions for everybody.\n\n" +
+ "Please assign your highest priority to fixing this. If you need support, request it using " +
+ "[yo/vespa-support|http://yo/vespa-support]. " +
+ "If this application is not in use, please re-assign this issue to project \"VESPA\" " +
+ "with component \"Services\", and ask for the application to be removed.\n\n" +
+ "If we do not get a response on this issue, we will auto-escalate it.";
+ }
+
+ /** Returns the comment text that what will be added to an issue each time it is escalated */
+ private String deploymentIssueEscalationComment(Application application, long propertyId, String priorAssignee) {
+ return "This issue tracks the failing deployment of Vespa application " +
+ "[" + toShortString(application.id()) + "|" + toUrl(application.id()) + "]. " +
+ "Since we have not received a response from " + priorAssignee +
+ ", we are escalating to you, " +
+ "based on [your OpsDb information|" + toOpsDbUrl(propertyId) + "]. " +
+ "Please acknowledge this issue and assign somebody to " +
+ "fix it as soon as possible.\n\n" +
+ "If we do not receive a response we will keep auto-escalating this issue. " +
+ "If we run out of escalation options for your OpsDb property, we will assume this application " +
+ "is not managed by anyone and DELETE it. In the meantime, this issue will block global deployment " +
+ "of Vespa for the entire company.";
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java
new file mode 100644
index 00000000000..9e8f902a8db
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Attempts redeployment of failed jobs and deployments.
+ *
+ * @author bratseth
+ */
+public class FailureRedeployer extends Maintainer {
+
+ public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) {
+ super(controller, interval, jobControl);
+ }
+
+ @Override
+ public void maintain() {
+ ApplicationList applications = ApplicationList.from(controller().applications().asList()).isDeploying();
+ List<Application> toTrigger = new ArrayList<>();
+
+ // Applications with deployment failures for current change and no running jobs
+ toTrigger.addAll(applications.hasDeploymentFailures()
+ .notRunningJob()
+ .asList());
+
+ // Applications with jobs that have been in progress for more than 12 hours
+ Instant twelveHoursAgo = controller().clock().instant().minus(Duration.ofHours(12));
+ toTrigger.addAll(applications.jobRunningSince(twelveHoursAgo).asList());
+
+ toTrigger.forEach(application -> controller().applications().deploymentTrigger()
+ .triggerFailing(application.id()));
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java
new file mode 100644
index 00000000000..e05612aaf57
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java
@@ -0,0 +1,67 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.logging.Logger;
+
+/**
+ * Provides status and control over running maintenance jobs.
+ * This is multithread safe.
+ *
+ * Job deactivation is stored in a local file.
+ *
+ * @author bratseth
+ */
+public class JobControl {
+
+ private static final Logger log = Logger.getLogger(JobControl.class.getName());
+
+ private final CuratorDb curator;
+
+ /** This is not stored in ZooKeeper as all nodes start all jobs */
+ private final Set<String> startedJobs = new ConcurrentSkipListSet<>();
+
+ /** Create a job control instance which persists activation changes to the default directory */
+ public JobControl(CuratorDb curator) {
+ this.curator = curator;
+ }
+
+ public CuratorDb curator() { return curator; }
+
+ /** Notifies this that a job was started */
+ public void started(String jobSimpleClassName) {
+ startedJobs.add(jobSimpleClassName);
+ }
+
+ /**
+ * Returns a snapshot of the set of jobs started on this system (whether deactivated or not).
+ * Each job is represented by its simple (omitting package) class name.
+ */
+ public Set<String> jobs() { return new HashSet<>(startedJobs); }
+
+ /** Returns an unmodifiable set containing the currently inactive jobs in this */
+ public Set<String> inactiveJobs() { return curator.readInactiveJobs(); }
+
+ /** Returns true if this job is not currently deactivated */
+ public boolean isActive(String jobSimpleClassName) {
+ return ! inactiveJobs().contains(jobSimpleClassName);
+ }
+
+ /** Set a job active or inactive */
+ public void setActive(String jobSimpleClassName, boolean active) {
+ try (Lock lock = curator.lockInactiveJobs()) {
+ Set<String> inactiveJobs = curator.readInactiveJobs();
+ if (active)
+ inactiveJobs.remove(jobSimpleClassName);
+ else
+ inactiveJobs.add(jobSimpleClassName);
+ curator.writeInactiveJobs(inactiveJobs);
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
new file mode 100644
index 00000000000..9f9f0175230
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
@@ -0,0 +1,80 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.time.Duration;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller.
+ *
+ * @author bratseth
+ */
+public abstract class Maintainer extends AbstractComponent implements Runnable {
+
+ protected static final Logger log = Logger.getLogger(Maintainer.class.getName());
+
+ private final Controller controller;
+ private final Duration maintenanceInterval;
+ private final JobControl jobControl;
+ private final ScheduledExecutorService service;
+
+ public Maintainer(Controller controller, Duration interval, JobControl jobControl) {
+ this.controller = controller;
+ this.maintenanceInterval = interval;
+ this.jobControl = jobControl;
+
+ service = new ScheduledThreadPoolExecutor(1);
+ service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS);
+ jobControl.started(name());
+ }
+
+ protected Controller controller() { return controller; }
+
+ protected CuratorDb curator() { return jobControl.curator(); }
+
+ @Override
+ public void run() {
+ try {
+ if (jobControl.isActive(name())) {
+ try (Lock lock = jobControl.curator().lockMaintenanceJob(name())) {
+ maintain();
+ }
+ }
+ }
+ catch (UncheckedTimeoutException e) {
+ // another controller instance is running this job at the moment; ok
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, this + " failed. Will retry in " + maintenanceInterval.toMinutes() + " minutes", e);
+ }
+ }
+
+ @Override
+ public void deconstruct() {
+ this.service.shutdown();
+ }
+
+ /** Called once each time this maintenance job should run */
+ protected abstract void maintain();
+
+ public Duration maintenanceInterval() { return maintenanceInterval; }
+
+ public String name() { return this.getClass().getSimpleName(); }
+
+ /** Returns the name of this */
+ @Override
+ public final String toString() {
+ return name();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
new file mode 100644
index 00000000000..3d0cd284c55
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -0,0 +1,118 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author mortent
+ */
+public class MetricsReporter extends Maintainer {
+
+ public static final String convergeMetric = "seconds.since.last.chef.convergence";
+ public static final String deploymentFailMetric = "deployment.failurePercentage";
+ private final Metric metric;
+ private final Chef chefClient;
+ private final Clock clock;
+ private final SystemName system;
+
+ public MetricsReporter(Controller controller, Metric metric, Chef chefClient, JobControl jobControl,
+ SystemName system) {
+ this(controller, metric, chefClient, Clock.systemUTC(), jobControl, system);
+ }
+
+ public MetricsReporter(Controller controller, Metric metric, Chef chefClient, Clock clock,
+ JobControl jobControl, SystemName system) {
+ super(controller, Duration.ofMinutes(1), jobControl); // use fixed rate for metrics
+ this.metric = metric;
+ this.chefClient = chefClient;
+ this.clock = clock;
+ this.system = system;
+ }
+
+ @Override
+ public void maintain() {
+ reportChefMetrics();
+ reportDeploymentMetrics();
+ }
+
+ private void reportChefMetrics() {
+ String query = "chef_environment:hosted*";
+ if (system == SystemName.cd) {
+ query += " AND hosted_system:" + system;
+ }
+ PartialNodeResult nodeResult = chefClient.partialSearchNodes(query,
+ Arrays.asList(
+ AttributeMapping.simpleMapping("fqdn"),
+ AttributeMapping.simpleMapping("ohai_time"),
+ AttributeMapping.deepMapping("tenant", Arrays.asList("hosted", "owner", "tenant")),
+ AttributeMapping.deepMapping("application", Arrays.asList("hosted", "owner", "application")),
+ AttributeMapping.deepMapping("instance", Arrays.asList("hosted", "owner", "instance")),
+ AttributeMapping.deepMapping("environment", Arrays.asList("hosted", "environment")),
+ AttributeMapping.deepMapping("region", Arrays.asList("hosted", "region")),
+ AttributeMapping.deepMapping("system", Arrays.asList("hosted", "system"))
+ ));
+
+ // The above search will return a correct list if the system is CD. However for main, it will
+ // return all nodes, since system==nil for main
+ keepNodesWithSystem(nodeResult, system);
+
+ Instant instant = clock.instant();
+ for (PartialNode node : nodeResult.rows) {
+ String hostname = node.getFqdn();
+ long secondsSinceConverge = Duration.between(Instant.ofEpochSecond(node.getOhaiTime().longValue()), instant).getSeconds();
+ Map<String, String> dimensions = new HashMap<>();
+ dimensions.put("host", hostname);
+ dimensions.put("system", node.getValue("system").orElse("main"));
+ Optional<String> environment = node.getValue("environment");
+ Optional<String> region = node.getValue("region");
+
+ if(environment.isPresent() && region.isPresent()) {
+ dimensions.put("zone", String.format("%s.%s", environment.get(), region.get()));
+ }
+
+ node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant));
+ Optional<String> application = node.getValue("application");
+ if (application.isPresent()) {
+ dimensions.put("app",String.format("%s.%s", application.get(), node.getValue("instance").orElse("default")));
+ }
+ Metric.Context context = metric.createContext(dimensions);
+ metric.set(convergeMetric, secondsSinceConverge, context);
+ }
+ }
+
+ private void reportDeploymentMetrics() {
+ metric.set(deploymentFailMetric, deploymentFailRatio() * 100, metric.createContext(Collections.emptyMap()));
+ }
+
+ private double deploymentFailRatio() {
+ List<Application> applications = controller().applications().asList();
+ if (applications.isEmpty()) return 0;
+
+ return (double)applications.stream().filter(a -> a.deploymentJobs().hasFailures()).count() /
+ (double)applications.size();
+ }
+
+ private void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) {
+ nodeResult.rows.removeIf(node -> !system.name().equals(node.getValue("system").orElse("main")));
+ }
+
+}
+
+
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
new file mode 100644
index 00000000000..4485a603f61
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
@@ -0,0 +1,32 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+import com.yahoo.vespa.hosted.controller.application.Change;
+
+import java.time.Duration;
+
+/**
+ * Deploys application changes which have been postponed due to an ongoing upgrade
+ *
+ * @author bratseth
+ */
+public class OutstandingChangeDeployer extends Maintainer {
+
+ public OutstandingChangeDeployer(Controller controller, Duration interval, JobControl jobControl) {
+ super(controller, interval, jobControl);
+ }
+
+ @Override
+ protected void maintain() {
+ ApplicationList applications = ApplicationList.from(controller().applications().asList()).notPullRequest();
+ for (Application application : applications.asList()) {
+ if (application.hasOutstandingChange() && ! application.deploying().isPresent())
+ controller().applications().deploymentTrigger().triggerChange(application.id(),
+ Change.ApplicationChange.unknown());
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
new file mode 100644
index 00000000000..b3d75106d2f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -0,0 +1,94 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
+import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Maintenance job which schedules applications for Vespa version upgrade
+ *
+ * @author bratseth
+ */
+public class Upgrader extends Maintainer {
+
+ private static final Logger log = Logger.getLogger(Upgrader.class.getName());
+
+ public Upgrader(Controller controller, Duration interval, JobControl jobControl) {
+ super(controller, interval, jobControl);
+ }
+
+ /**
+ * Schedule application upgrades. Note that this implementation must be idempotent.
+ */
+ @Override
+ public void maintain() {
+ VespaVersion target = controller().versionStatus().version(controller().systemVersion());
+ if (target == null) return; // we don't have information about the current system version at this time
+
+ // TODO: Remove corp-prod special casing when corp-prod and main are upgraded at the same time
+ if (Vtag.currentVersion.isAfter(target.versionNumber())) {
+ upgrade(applications().deploysTo(Environment.prod, RegionName.from("corp-us-east-1")).with(UpgradePolicy.canary),
+ Vtag.currentVersion);
+ }
+
+ switch (target.confidence()) {
+ case broken:
+ log.info(String.format("Version %s is broken, cancelling all upgrades", target.versionNumber()));
+ cancelUpgradesOf(applications().upgradingTo(target.versionNumber())
+ .without(UpgradePolicy.canary)); // keep trying canaries
+ break;
+ case low:
+ upgrade(applications().with(UpgradePolicy.canary), target.versionNumber());
+ break;
+ case normal:
+ upgrade(applications().with(UpgradePolicy.defaultPolicy), target.versionNumber());
+ break;
+ case high:
+ upgrade(applications().with(UpgradePolicy.conservative), target.versionNumber());
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown version confidence " + target.confidence());
+ }
+ }
+
+ /** Returns a list of all applications */
+ private ApplicationList applications() { return ApplicationList.from(controller().applications().asList()); }
+
+ private void upgrade(ApplicationList applications, Version version) {
+ Change.VersionChange change = new Change.VersionChange(version);
+ cancelUpgradesOf(applications.upgradingToLowerThan(version));
+ applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade
+ applications = applications.onLowerVersionThan(version);
+ applications = applications.notDeployingApplication(); // wait with applications deploying an application change
+ applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version
+ applications = applications.notRunningJobFor(change); // do not trigger multiple jobs simultaneously for same upgrade
+ for (Application application : applications.byIncreasingDeployedVersion().asList()) {
+ try {
+ controller().applications().deploymentTrigger().triggerChange(application.id(), change);
+ } catch (IllegalArgumentException e) {
+ log.log(Level.INFO, "Could not trigger change: " + Exceptions.toMessageString(e));
+ }
+ }
+ }
+
+ private void cancelUpgradesOf(ApplicationList applications) {
+ for (Application application : applications.asList()) {
+ controller().applications().deploymentTrigger().cancelChange(application.id());
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
new file mode 100644
index 00000000000..dea991bc653
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+
+import java.io.UncheckedIOException;
+import java.time.Duration;
+
+/**
+ * This maintenance job periodically updates the version status.
+ * Since the version status is expensive to compute and do not need to be perfectly up to date,
+ * we do not want to recompute it each time it is accessed.
+ *
+ * @author bratseth
+ */
+public class VersionStatusUpdater extends Maintainer {
+
+ public VersionStatusUpdater(Controller controller, Duration interval, JobControl jobControl) {
+ super(controller, interval, jobControl);
+ }
+
+ @Override
+ protected void maintain() {
+ try {
+ VersionStatus newStatus = VersionStatus.compute(controller());
+ controller().updateVersionStatus(newStatus);
+ } catch (UncheckedIOException e) {
+ log.warning("Failed to compute version status. This is likely a transient error: " + e.getMessage());
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java
new file mode 100644
index 00000000000..14267807041
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java
new file mode 100644
index 00000000000..112e90e2cd7
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * The root package of the controller
+ *
+ * @author bratseth
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
new file mode 100644
index 00000000000..014c63a6779
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -0,0 +1,304 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Serializes applications to/from slime.
+ * This class is multithread safe.
+ *
+ * @author bratseth
+ */
+public class ApplicationSerializer {
+
+ // Application fields
+ private final String idField = "id";
+ private final String deploymentSpecField = "deploymentSpecField";
+ private final String validationOverridesField = "validationOverrides";
+ private final String deploymentsField = "deployments";
+ private final String deploymentJobsField = "deploymentJobs";
+ private final String deployingField = "deployingField";
+ private final String outstandingChangeField = "outstandingChangeField";
+
+ // Deployment fields
+ private final String zoneField = "zone";
+ private final String environmentField = "environment";
+ private final String regionField = "region";
+ private final String deployTimeField = "deployTime";
+ private final String applicationPackageRevisionField = "applicationPackageRevision";
+ private final String applicationPackageHashField = "applicationPackageHash";
+ private final String sourceRevisionField = "sourceRevision";
+ private final String repositoryField = "repositoryField";
+ private final String branchField = "branchField";
+ private final String commitField = "commitField";
+
+ // DeploymentJobs fields
+ private final String projectIdField = "projectId";
+ private final String jobStatusField = "jobStatus";
+ private final String jiraIssueIdField = "jiraIssueId";
+ private final String selfTriggeringField = "selfTriggering";
+
+ // JobStatus field
+ private final String jobTypeField = "jobType";
+ private final String errorField = "jobError";
+ private final String completionTimeField = "completionTime";
+ private final String failingSinceField = "failingSince";
+ private final String lastTriggeredField = "lastTriggered";
+ private final String lastCompletedField = "lastCompleted";
+ private final String firstFailingField = "firstFailing";
+ private final String lastSuccessField = "lastSuccess";
+
+ // JobRun fields
+ private final String versionField = "version";
+ private final String revisionField = "revision";
+ private final String atField = "at";
+
+ // ------------------ Serialization
+
+ public Slime toSlime(Application application) {
+ Slime slime = new Slime();
+ Cursor root = slime.setObject();
+ root.setString(idField, application.id().serializedForm());
+ root.setString(deploymentSpecField, application.deploymentSpec().xmlForm());
+ root.setString(validationOverridesField, application.validationOverrides().xmlForm());
+ deploymentsToSlime(application.deployments().values(), root.setArray(deploymentsField));
+ toSlime(application.deploymentJobs(), root.setObject(deploymentJobsField));
+ toSlime(application.deploying(), root);
+ root.setBool(outstandingChangeField, application.hasOutstandingChange());
+ return slime;
+ }
+
+ private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) {
+ for (Deployment deployment : deployments)
+ deploymentToSlime(deployment, array.addObject());
+ }
+
+ private void deploymentToSlime(Deployment deployment, Cursor object) {
+ zoneToSlime(deployment.zone(), object.setObject(zoneField));
+ object.setString(versionField, deployment.version().toString());
+ object.setLong(deployTimeField, deployment.at().toEpochMilli());
+ toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField));
+ }
+
+ private void zoneToSlime(Zone zone, Cursor object) {
+ object.setString(environmentField, zone.environment().value());
+ object.setString(regionField, zone.region().value());
+ }
+
+ private void toSlime(ApplicationRevision applicationRevision, Cursor object) {
+ object.setString(applicationPackageHashField, applicationRevision.id());
+ if (applicationRevision.source().isPresent())
+ toSlime(applicationRevision.source().get(), object.setObject(sourceRevisionField));
+ }
+
+ private void toSlime(SourceRevision sourceRevision, Cursor object) {
+ object.setString(repositoryField, sourceRevision.repository());
+ object.setString(branchField, sourceRevision.branch());
+ object.setString(commitField, sourceRevision.commit());
+ }
+
+ private void toSlime(DeploymentJobs deploymentJobs, Cursor cursor) {
+ deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId));
+ jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField));
+ deploymentJobs.jiraIssueId().ifPresent(jiraIssueId -> cursor.setString(jiraIssueIdField, jiraIssueId));
+ cursor.setBool(selfTriggeringField, deploymentJobs.isSelfTriggering());
+ }
+
+ private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) {
+ for (JobStatus jobStatus : jobStatuses)
+ toSlime(jobStatus, jobStatusArray.addObject());
+ }
+
+ private void toSlime(JobStatus jobStatus, Cursor object) {
+ object.setString(jobTypeField, jobStatus.type().id());
+ if (jobStatus.jobError().isPresent())
+ object.setString(errorField, jobStatus.jobError().get().name());
+
+ jobRunToSlime(jobStatus.lastTriggered(), object, lastTriggeredField);
+ jobRunToSlime(jobStatus.lastCompleted(), object, lastCompletedField);
+ jobRunToSlime(jobStatus.firstFailing(), object, firstFailingField);
+ jobRunToSlime(jobStatus.lastSuccess(), object, lastSuccessField);
+ }
+
+ private void jobRunToSlime(Optional<JobStatus.JobRun> jobRun, Cursor parent, String jobRunObjectName) {
+ if ( ! jobRun.isPresent()) return;
+ Cursor object = parent.setObject(jobRunObjectName);
+ object.setString(versionField, jobRun.get().version().toString());
+ if ( jobRun.get().revision().isPresent())
+ toSlime(jobRun.get().revision().get(), object.setObject(revisionField));
+ object.setLong(atField, jobRun.get().at().toEpochMilli());
+ }
+
+ private void toSlime(Optional<Change> deploying, Cursor parentObject) {
+ if ( ! deploying.isPresent()) return;
+
+ Cursor object = parentObject.setObject(deployingField);
+ if (deploying.get() instanceof Change.VersionChange)
+ object.setString(versionField, ((Change.VersionChange)deploying.get()).version().toString());
+ else if (((Change.ApplicationChange)deploying.get()).revision().isPresent())
+ toSlime(((Change.ApplicationChange)deploying.get()).revision().get(), object);
+ }
+
+ // ------------------ Deserialization
+
+ public Application fromSlime(Slime slime) {
+ Inspector root = slime.get();
+
+ ApplicationId id = ApplicationId.fromSerializedForm(root.field(idField).asString());
+ DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString());
+ ValidationOverrides validationOverrides = validationOverridesFromSlime(root.field(validationOverridesField));
+ List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField));
+ DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField));
+ Optional<Change> deploying = changeFromSlime(root.field(deployingField));
+ boolean outstandingChange = root.field(outstandingChangeField).asBool();
+
+ return new Application(id, deploymentSpec, validationOverrides, deployments,
+ deploymentJobs, deploying, outstandingChange);
+ }
+
+ private ValidationOverrides validationOverridesFromSlime(Inspector field) {
+ if ( ! field.valid()) return ValidationOverrides.empty; // TODO: Remove this line (and inline function) after June 2017
+ return ValidationOverrides.fromXml(field.asString());
+ }
+
+ private List<Deployment> deploymentsFromSlime(Inspector array) {
+ List<Deployment> deployments = new ArrayList<>();
+ array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item)));
+ return deployments;
+ }
+
+ private Deployment deploymentFromSlime(Inspector deploymentObject) {
+ return new Deployment(zoneFromSlime(deploymentObject.field(zoneField)),
+ applicationRevisionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(),
+ Version.fromString(deploymentObject.field(versionField).asString()),
+ Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()));
+ }
+
+ private Zone zoneFromSlime(Inspector object) {
+ return new Zone(Environment.from(object.field(environmentField).asString()),
+ RegionName.from(object.field(regionField).asString()));
+ }
+
+ private Optional<ApplicationRevision> applicationRevisionFromSlime(Inspector object) {
+ if ( ! object.valid()) return Optional.empty();
+ String applicationPackageHash = object.field(applicationPackageHashField).asString();
+ Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField));
+ return sourceRevision.isPresent() ? Optional.of(ApplicationRevision.from(applicationPackageHash, sourceRevision.get()))
+ : Optional.of(ApplicationRevision.from(applicationPackageHash));
+ }
+
+ private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) {
+ if ( ! object.valid()) return Optional.empty();
+ return Optional.of(new SourceRevision(object.field(repositoryField).asString(),
+ object.field(branchField).asString(),
+ object.field(commitField).asString()));
+ }
+
+ private DeploymentJobs deploymentJobsFromSlime(Inspector object) {
+ Optional<Long> projectId = optionalLong(object.field(projectIdField));
+ List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField));
+ Optional<String> jiraIssueKey = optionalString(object.field(jiraIssueIdField));
+ boolean selfTriggering = object.field(selfTriggeringField).asBool();
+
+ return new DeploymentJobs(projectId, jobStatusList, jiraIssueKey, selfTriggering);
+ }
+
+ private Optional<Change> changeFromSlime(Inspector object) {
+ if ( ! object.valid()) return Optional.empty();
+ Inspector versionFieldValue = object.field(versionField);
+ if (versionFieldValue.valid())
+ return Optional.of(new Change.VersionChange(Version.fromString(versionFieldValue.asString())));
+ else if (object.field(applicationPackageHashField).valid())
+ return Optional.of(Change.ApplicationChange.of(applicationRevisionFromSlime(object).get()));
+ else
+ return Optional.of(Change.ApplicationChange.unknown());
+ }
+
+ private List<JobStatus> jobStatusListFromSlime(Inspector array) {
+ List<JobStatus> jobStatusList = new ArrayList<>();
+ array.traverse((ArrayTraverser) (int i, Inspector item) -> {
+ // TODO: This zone has been removed. Remove after Aug 2017
+ String jobId = item.field(jobTypeField).asString();
+ if ("production-ap-aue-1".equals(jobId)) {
+ return;
+ }
+ jobStatusList.add(jobStatusFromSlime(item));
+ });
+ return jobStatusList;
+ }
+
+ private JobStatus jobStatusFromSlime(Inspector object) {
+ DeploymentJobs.JobType jobType = DeploymentJobs.JobType.fromId(object.field(jobTypeField).asString());
+
+ Optional<JobError> jobError = Optional.empty();
+ if (object.field(errorField).valid())
+ jobError = Optional.of(JobError.valueOf(object.field(errorField).asString()));
+
+ Inspector versionFieldValue = object.field(versionField);
+ if (versionFieldValue.valid()) { // TODO: Read legacy JobStatus content: Remove after June 2017
+ // Read stored information in old data model
+ Instant completionTime = Instant.ofEpochMilli(object.field(completionTimeField).asLong());
+ Optional<Instant> failingSinceTime = optionalLong(object.field(failingSinceField)).map(Instant::ofEpochMilli);
+ Optional<Instant> lastTriggeredTime = optionalLong(object.field(lastTriggeredField)).map(Instant::ofEpochMilli);
+ Version version = new Version(versionFieldValue.asString());
+
+ // Best-effort conversion to new data model
+ Optional<JobStatus.JobRun> lastTriggered = lastTriggeredTime.map(at -> new JobStatus.JobRun(version, Optional.empty(), at));
+ Optional<JobStatus.JobRun> lastCompleted = Optional.of(new JobStatus.JobRun(version, Optional.empty(), completionTime));
+ Optional<JobStatus.JobRun> firstFailing = failingSinceTime.map(at -> new JobStatus.JobRun(version, Optional.empty(), at));
+ Optional<JobStatus.JobRun> lastSuccess = Optional.of(new JobStatus.JobRun(version, Optional.empty(), completionTime));;
+
+ return new JobStatus(jobType, jobError,
+ lastTriggered, lastCompleted, firstFailing, lastSuccess);
+ }
+ else { // read current format
+ return new JobStatus(jobType, jobError,
+ jobRunFromSlime(object.field(lastTriggeredField)),
+ jobRunFromSlime(object.field(lastCompletedField)),
+ jobRunFromSlime(object.field(firstFailingField)),
+ jobRunFromSlime(object.field(lastSuccessField)));
+
+ }
+ }
+
+ private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) {
+ if ( ! object.valid()) return Optional.empty();
+ return Optional.of(new JobStatus.JobRun(new Version(object.field(versionField).asString()),
+ applicationRevisionFromSlime(object.field(revisionField)),
+ Instant.ofEpochMilli(object.field(atField).asLong())));
+ }
+
+ private Optional<Long> optionalLong(Inspector field) {
+ return field.valid() ? Optional.of(field.asLong()) : Optional.empty();
+ }
+
+ private Optional<String> optionalString(Inspector field) {
+ return SlimeUtils.optionalString(field);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java
new file mode 100644
index 00000000000..3fbfdd31808
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java
@@ -0,0 +1,74 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.google.common.base.Joiner;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Used to store the permanent data of the controller.
+ *
+ * @author Stian Kristoffersen
+ * @author bratseth
+ */
+public abstract class ControllerDb {
+
+ // --------- Tenants
+
+ public abstract void createTenant(Tenant tenant);
+
+ public abstract void updateTenant(Tenant tenant) throws PersistenceException;
+
+ public abstract void deleteTenant(TenantId tenantId) throws PersistenceException;
+
+ public abstract Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException;
+
+ public abstract List<Tenant> listTenants();
+
+ // --------- Applications
+
+ // ONLY call this from ApplicationController.store()
+ public abstract void store(Application application);
+
+ public abstract void deleteApplication(ApplicationId applicationId);
+
+ public abstract Optional<Application> getApplication(ApplicationId applicationId);
+
+ /** Returns all applications */
+ public abstract List<Application> listApplications();
+
+ /** Returns all applications of a tenant */
+ public abstract List<Application> listApplications(TenantId tenantId);
+
+ // --------- Rotations
+
+ public abstract Set<RotationId> getRotations();
+
+ public abstract Set<RotationId> getRotations(ApplicationId applicationId);
+
+ public abstract boolean assignRotation(RotationId rotationId, ApplicationId applicationId);
+
+ public abstract Set<RotationId> deleteRotations(ApplicationId applicationId);
+
+ /** Returns the given elements joined by dot "." */
+ protected String path(Identifier... elements) {
+ return Joiner.on(".").join(elements);
+ }
+
+ protected String path(String... elements) {
+ return Joiner.on(".").join(elements);
+ }
+
+ protected String path(ApplicationId applicationId) {
+ return applicationId.tenant().value() + "." + applicationId.application().value() + "." + applicationId.instance().value();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
new file mode 100644
index 00000000000..5777636fa24
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -0,0 +1,201 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.path.Path;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Curator backed database for storing working state shared between controller servers.
+ * This maps controller specific operations to general curator operations.
+ *
+ * @author bratseth
+ */
+public class CuratorDb {
+
+ private static final Logger log = Logger.getLogger(CuratorDb.class.getName());
+
+ private static final Path root = Path.fromString("/controller/v1");
+
+ private static final Duration defaultLockTimeout = Duration.ofMinutes(5);
+
+ private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
+ private final JobQueueSerializer jobQueueSerializer = new JobQueueSerializer();
+
+ @SuppressWarnings("unused") // This server is used (only) from the curator instance of this over the network */
+ //private final ZooKeeperServer zooKeeperServer;
+ private final Curator curator;
+
+ /**
+ * All keys, to allow reentrancy.
+ * This will grow forever, but this should be too slow to be a problem.
+ */
+ private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>();
+
+ /** Create a curator db which also set up a ZooKeeper server (such that this instance is both client and server) */
+ @Inject
+ public CuratorDb() {
+ // this.zooKeeperServer = new ZooKeeperServer(createZookeeperServerConfig());
+ // this.curator = new Curator("localhost:2281");
+ //this.zooKeeperServer = null;
+ this.curator = new MockCurator();
+ }
+
+ private static ZookeeperServerConfig createZookeeperServerConfig() {
+ ZookeeperServerConfig.Builder b = new ZookeeperServerConfig.Builder();
+ b.zooKeeperConfigFile("conf/zookeeper/controller-zookeeper.cfg");
+ b.dataDir("var/controller-zookeeper");
+ b.clientPort(2281);
+ b.myidFile("var/controller-zookeeper/myid");
+ b.myid(0);
+ ZookeeperServerConfig.Server.Builder server = new ZookeeperServerConfig.Server.Builder();
+ server.id(0);
+ server.hostname("localhost");
+ server.quorumPort(2282);
+ server.electionPort(2283);
+ b.server(server);
+ return new ZookeeperServerConfig(b);
+ }
+
+ /** Create a curator db which does not set uop a server, using the given Curator instance */
+ protected CuratorDb(Curator curator) {
+ //this.zooKeeperServer = null;
+ this.curator = curator;
+ }
+
+ // -------------- Locks --------------------------------------------------
+
+ public Lock lock(TenantId id, Duration timeout) {
+ return lock(lockPath(id), timeout);
+ }
+
+ public Lock lock(ApplicationId id, Duration timeout) {
+ return lock(lockPath(id), timeout);
+ }
+
+ /** Create a reentrant lock */
+ private Lock lock(Path path, Duration timeout) {
+ Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), curator));
+ lock.acquire(timeout);
+ return lock;
+ }
+
+ public Lock lockInactiveJobs() {
+ return lock(root.append("locks").append("inactiveJobsLock"), defaultLockTimeout);
+ }
+
+ public Lock lockJobQueues() {
+ return lock(root.append("locks").append("jobQueuesLock"), defaultLockTimeout);
+ }
+
+ public Lock lockMaintenanceJob(String jobName) {
+ // Use a short timeout such that if maintenance jobs are started at about the same time on different nodes
+ // and the maintenance job takes a long time to complete, only one of the nodes will run the job
+ // in each maintenance interval
+ return lock(root.append("locks").append("maintenanceJobLocks").append(jobName), Duration.ofSeconds(1));
+ }
+
+ // -------------- Read and write --------------------------------------------------
+
+ public Version readSystemVersion() {
+ Optional<byte[]> data = curator.getData(systemVersionPath());
+ if (! data.isPresent() || data.get().length == 0) return Vtag.currentVersion;
+ return Version.fromString(new String(data.get(), StandardCharsets.UTF_8));
+ }
+
+ public void writeSystemVersion(Version version) {
+ NestedTransaction transaction = new NestedTransaction();
+ curator.set(systemVersionPath(), version.toString().getBytes(StandardCharsets.UTF_8));
+ transaction.commit();
+ }
+
+ public Set<String> readInactiveJobs() {
+ try {
+ Optional<byte[]> data = curator.getData(inactiveJobsPath());
+ if (! data.isPresent() || data.get().length == 0) return new HashSet<>(); // inactive jobs has never been written
+ return stringSetSerializer.fromJson(data.get());
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Error reading inactive jobs, deleting inactive state");
+ writeInactiveJobs(Collections.emptySet());
+ return new HashSet<>();
+ }
+ }
+
+ public void writeInactiveJobs(Set<String> inactiveJobs) {
+ NestedTransaction transaction = new NestedTransaction();
+ curator.set(inactiveJobsPath(), stringSetSerializer.toJson(inactiveJobs));
+ transaction.commit();
+ }
+
+ public Deque<ApplicationId> readJobQueue(DeploymentJobs.JobType jobType) {
+ try {
+ Optional<byte[]> data = curator.getData(jobQueuePath(jobType));
+ if (! data.isPresent() || data.get().length == 0) return new ArrayDeque<>(); // job queue has never been written
+ return jobQueueSerializer.fromJson(data.get());
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Error reading job queue, deleting inactive state");
+ writeInactiveJobs(Collections.emptySet());
+ return new ArrayDeque<>();
+ }
+ }
+
+ public void writeJobQueue(DeploymentJobs.JobType jobType, Deque<ApplicationId> queue) {
+ NestedTransaction transaction = new NestedTransaction();
+ curator.set(jobQueuePath(jobType), jobQueueSerializer.toJson(queue));
+ transaction.commit();
+ }
+
+ // -------------- Paths --------------------------------------------------
+
+ private Path systemVersionPath() {
+ return root.append("systemVersion");
+ }
+
+ private Path lockPath(TenantId tenant) {
+ Path lockPath = root.append("locks")
+ .append(tenant.id());
+ curator.create(lockPath);
+ return lockPath;
+ }
+
+ private Path lockPath(ApplicationId application) {
+ Path lockPath = root.append("locks")
+ .append(application.tenant().value())
+ .append(application.application().value())
+ .append(application.instance().value());
+ curator.create(lockPath);
+ return lockPath;
+ }
+
+ private Path inactiveJobsPath() {
+ return root.append("inactiveJobs");
+ }
+
+ private Path jobQueuePath(DeploymentJobs.JobType jobType) {
+ return root.append("jobQueues").append(jobType.name());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java
new file mode 100644
index 00000000000..5017624f286
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/JobQueueSerializer.java
@@ -0,0 +1,45 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Serialization of a queue of ApplicationIds to/from Json bytes using Slime.
+ *
+ * The set is serialized as an array of string.
+ *
+ * @author bratseth
+ */
+public class JobQueueSerializer {
+
+ public byte[] toJson(Deque<ApplicationId> queue) {
+ try {
+ Slime slime = new Slime();
+ Cursor array = slime.setArray();
+ queue.forEach((id -> array.addString(id.serializedForm())));
+ return SlimeUtils.toJsonBytes(slime);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Serialization of a job queue failed", e);
+ }
+ }
+
+ public Deque<ApplicationId> fromJson(byte[] data) {
+ Inspector inspector = SlimeUtils.jsonToSlime(data).get();
+ Deque<ApplicationId> queue = new ArrayDeque<>();
+ inspector.traverse((ArrayTraverser) (index, value) -> queue.addLast(ApplicationId.fromSerializedForm(value.asString())));
+ return queue;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java
new file mode 100644
index 00000000000..37677a5e393
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java
@@ -0,0 +1,132 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.AlreadyExistsException;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.NotExistsException;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A controller db implementation backed by in-memory structures. Useful for testing.
+ *
+ * @author Stian Kristoffersen
+ */
+public class MemoryControllerDb extends ControllerDb {
+
+ private Map<TenantId, Tenant> tenants = new HashMap<>();
+ private Map<String, Application> applications = new HashMap<>();
+ private Map<RotationId, ApplicationId> rotationAssignments = new HashMap<>();
+
+ @Override
+ public void createTenant(Tenant tenant) {
+ if (tenants.containsKey(tenant.getId())) {
+ throw new AlreadyExistsException(tenant.getId());
+ }
+ tenants.put(tenant.getId(), tenant);
+ }
+
+ @Override
+ public void updateTenant(Tenant tenant) {
+ if (!tenants.containsKey(tenant.getId())) {
+ throw new NotExistsException(tenant.getId());
+ }
+ tenants.put(tenant.getId(), tenant);
+ }
+
+ @Override
+ public void deleteTenant(TenantId tenantId) {
+ Object removed = tenants.remove(tenantId);
+ if (removed == null)
+ throw new NotExistsException(tenantId);
+ }
+
+ @Override
+ public Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException {
+ Optional<Tenant> tenant = Optional.ofNullable(tenants.get(tenantId));
+ if(tenant.isPresent()) {
+ Tenant t_noquota = tenant.get();
+ Tenant t_withquota = new Tenant(
+ t_noquota.getId(), t_noquota.getUserGroup(), t_noquota.getProperty(),
+ t_noquota.getAthensDomain(), t_noquota.getPropertyId());
+ return Optional.of(t_withquota);
+ } else {
+ return tenant;
+ }
+ }
+
+ @Override
+ public List<Tenant> listTenants() {
+ return new ArrayList<>(tenants.values());
+ }
+
+ @Override
+ public void store(Application application) {
+ applications.put(path(application.id()), application);
+ }
+
+ @Override
+ public void deleteApplication(ApplicationId applicationId) {
+ applications.remove(path(applicationId));
+ }
+
+ @Override
+ public Optional<Application> getApplication(ApplicationId applicationId) {
+ return Optional.ofNullable(applications.get(path(applicationId)));
+ }
+
+ @Override
+ public List<Application> listApplications() {
+ return new ArrayList<>(applications.values());
+ }
+
+ @Override
+ public List<Application> listApplications(TenantId tenantId) {
+ return applications.values().stream()
+ .filter(a -> a.id().tenant().value().equals(tenantId.id()))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public Set<RotationId> getRotations() {
+ return rotationAssignments.keySet();
+ }
+
+ @Override
+ public Set<RotationId> getRotations(ApplicationId applicationId) {
+ return rotationAssignments.entrySet().stream()
+ .filter(entry -> entry.getValue().equals(applicationId))
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toSet());
+ }
+
+ @Override
+ public boolean assignRotation(RotationId rotationId, ApplicationId applicationId) {
+ if (rotationAssignments.containsKey(rotationId)) {
+ return false;
+ } else {
+ rotationAssignments.put(rotationId, applicationId);
+ return true;
+ }
+ }
+
+ @Override
+ public Set<RotationId> deleteRotations(ApplicationId applicationId) {
+ Set<RotationId> rotations = getRotations(applicationId);
+ for (RotationId rotation : rotations) {
+ rotationAssignments.remove(rotation);
+ }
+ return rotations;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
new file mode 100644
index 00000000000..5dc8ca0e545
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.vespa.curator.mock.MockCurator;
+
+/**
+ * A curator db backed by a mock curator.
+ *
+ * @author bratseth
+ */
+@SuppressWarnings("unused") // injected
+public class MockCuratorDb extends CuratorDb {
+
+ public MockCuratorDb() {
+ super(new MockCurator());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java
new file mode 100644
index 00000000000..b963ecbfab9
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/PersistenceException.java
@@ -0,0 +1,19 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+/**
+ * Exception thrown by persistence layer.
+ *
+ * @author mpolden
+ */
+public class PersistenceException extends Exception {
+
+ public PersistenceException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public PersistenceException(Throwable cause) {
+ super(cause);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java
new file mode 100644
index 00000000000..83715e16e8e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Serialization of a set of strings to/from Json bytes using Slime.
+ *
+ * The set is serialized as an array of string.
+ *
+ * @author bratseth
+ */
+public class StringSetSerializer {
+
+ public byte[] toJson(Set<String> stringSet) {
+ try {
+ Slime slime = new Slime();
+ Cursor array = slime.setArray();
+ for (String element : stringSet)
+ array.addString(element);
+ return SlimeUtils.toJsonBytes(slime);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Serialization of a string set failed", e);
+ }
+
+ }
+
+ public Set<String> fromJson(byte[] data) {
+ Inspector inspector = SlimeUtils.jsonToSlime(data).get();
+ Set<String> stringSet = new HashSet<>();
+ inspector.traverse((ArrayTraverser) (index, name) -> stringSet.add(name.asString()));
+ return stringSet;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java
new file mode 100644
index 00000000000..87a14660fee
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Persistence layer for the controller.
+ *
+ * @author bratseth
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java
new file mode 100644
index 00000000000..a9643e21c00
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ErrorResponse.java
@@ -0,0 +1,66 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
+import com.yahoo.yolean.Exceptions;
+
+import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
+import static com.yahoo.jdisc.Response.Status.FORBIDDEN;
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED;
+import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
+
+/**
+ * A HTTP JSON response containing an error code and a message
+ *
+ * @author bratseth
+ */
+public class ErrorResponse extends SlimeJsonResponse {
+
+ public enum errorCodes {
+ NOT_FOUND,
+ BAD_REQUEST,
+ FORBIDDEN,
+ METHOD_NOT_ALLOWED,
+ INTERNAL_SERVER_ERROR
+ }
+
+ public ErrorResponse(int statusCode, String errorType, String message) {
+ super(statusCode, asSlimeMessage(errorType, message));
+ }
+
+ private static Slime asSlimeMessage(String errorType, String message) {
+ Slime slime = new Slime();
+ Cursor root = slime.setObject();
+ root.setString("error-code", errorType);
+ root.setString("message", message);
+ return slime;
+ }
+
+ public static ErrorResponse notFoundError(String message) {
+ return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message);
+ }
+
+ public static ErrorResponse internalServerError(String message) {
+ return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message);
+ }
+
+ public static ErrorResponse badRequest(String message) {
+ return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message);
+ }
+
+ public static ErrorResponse forbidden(String message) {
+ return new ErrorResponse(FORBIDDEN, errorCodes.FORBIDDEN.name(), message);
+ }
+
+ public static ErrorResponse methodNotAllowed(String message) {
+ return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message);
+ }
+
+ public static ErrorResponse from(ConfigServerException e) {
+ return new ErrorResponse(BAD_REQUEST, e.getErrorCode().name(), Exceptions.toMessageString(e));
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java
new file mode 100644
index 00000000000..8b2f0e9f09d
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/MessageResponse.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author bratseth
+ */
+public class MessageResponse extends HttpResponse {
+
+ private final Slime slime = new Slime();
+
+ public MessageResponse(String message) {
+ super(200);
+ slime.setObject().setString("message", message);
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java
new file mode 100644
index 00000000000..c8c027d91c9
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Path.java
@@ -0,0 +1,109 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A path which is able to match strings containing bracketed placeholders and return the
+ * values given at the placeholders.
+ *
+ * E.g a path /a/1/bar/fuz
+ * will match /a/{foo}/bar/{b}
+ * and return foo=1 and b=fuz
+ *
+ * Only full path elements may be placeholders, i.e /a{bar} is not interpreted as one.
+ *
+ * If the path spec ends with /{*}, it will match urls with any rest path.
+ * The rest path (not including the trailing slash) will be available as getRest().
+ *
+ * Note that for convenience in common use this has state which is changes as a side effect of each matches
+ * invocation. It is therefore for single thread use.
+ *
+ * @author bratseth
+ */
+public class Path {
+
+ // This path
+ private final String pathString;
+ private final String[] elements;
+
+ // Info about the last match
+ private final Map<String, String> values = new HashMap<>();
+ private String rest = "";
+
+ public Path(String path) {
+ this.pathString = path;
+ this.elements = path.split("/");
+ }
+
+ /**
+ * Returns whether this path matches the given template string.
+ * If the given template has placeholders, their values (accessible by get) are reset by calling this,
+ * whether or not the path matches the given template.
+ *
+ * This will NOT match empty path elements.
+ *
+ * @param pathSpec the path string to match to this
+ * @return true if the string matches, false otherwise
+ */
+ public boolean matches(String pathSpec) {
+ values.clear();
+ String[] specElements = pathSpec.split("/");
+ boolean matchPrefix = false;
+ if (specElements[specElements.length-1].equals("{*}")) {
+ matchPrefix = true;
+ specElements = Arrays.copyOf(specElements, specElements.length-1);
+ }
+
+ if (matchPrefix) {
+ if (this.elements.length < specElements.length) return false;
+ }
+ else { // match exact
+ if (this.elements.length != specElements.length) return false;
+ }
+
+ for (int i = 0; i < specElements.length; i++) {
+ if (specElements[i].startsWith("{") && specElements[i].endsWith("}")) // placeholder
+ values.put(specElements[i].substring(1, specElements[i].length()-1), elements[i]);
+ else if ( ! specElements[i].equals(this.elements[i]))
+ return false;
+ }
+
+ if (matchPrefix) {
+ StringBuilder rest = new StringBuilder();
+ for (int i = specElements.length; i < this.elements.length; i++)
+ rest.append(elements[i]).append("/");
+ if ( ! pathString.endsWith("/"))
+ rest.setLength(rest.length() -1);
+ this.rest = rest.toString();
+ }
+
+ return true;
+ }
+
+ /**
+ * Returns the value of the given template variable in the last path matched, or null
+ * if the previous matches call returned false or if this has not matched anything yet.
+ */
+ public String get(String placeholder) {
+ return values.get(placeholder);
+ }
+
+ /**
+ * Returns the rest of the last matched path.
+ * This is always the empty string (never null) unless the path spec ends with {*}
+ */
+ public String getRest() { return rest; }
+
+ /** Returns this path as a string */
+ public String asString() { return pathString; }
+
+ @Override
+ public String toString() {
+ return "path '" + Arrays.stream(elements).collect(Collectors.joining("/")) + "'";
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java
new file mode 100644
index 00000000000..550b47d8280
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/ResourceResponse.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Returns a response containing an array of links to sub-resources
+ *
+ * @author bratseth
+ */
+public class ResourceResponse extends HttpResponse {
+
+ private final Slime slime = new Slime();
+
+ public ResourceResponse(HttpRequest request, String ... subResources) {
+ super(200);
+ Cursor resourceArray = slime.setObject().setArray("resources");
+ for (String subResource : subResources) {
+ Cursor resourceEntry = resourceArray.addObject();
+ resourceEntry.setString("url", new Uri(request.getUri())
+ .append(subResource)
+ .withTrailingSlash()
+ .toString());
+ }
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java
new file mode 100644
index 00000000000..9283b1c3018
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/RootHandler.java
@@ -0,0 +1,96 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.concurrent.Executor;
+
+/**
+ * Responds to requests for the root path of the controller by listing the available web service API's.
+ *
+ * FAQ:
+ * - Q: Why do we need this when the container provides a perfectly fine root response listing all handlers by default?
+ * - A: Because we also have Jersey API's and those are not included in the default response.
+ *
+ * @author Oyvind Gronnesby
+ * @author bratseth
+ */
+public class RootHandler extends LoggingRequestHandler {
+
+ public RootHandler(Executor executor, AccessLog accessLog) {
+ super(executor, accessLog);
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest httpRequest) {
+ final URI requestUri = httpRequest.getUri();
+ return new ControllerRootPathResponse(requestUri);
+ }
+
+ private static class ControllerRootPathResponse extends HttpResponse {
+
+ private final URI uri;
+
+ public ControllerRootPathResponse(URI uri) {
+ super(200);
+ this.uri = uri;
+ }
+
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.writeValue(outputStream, buildResponseObject());
+ }
+
+ @Override
+ public String getContentType() {
+ return "application/json";
+ }
+
+ private JsonNode buildResponseObject() {
+ ObjectNode output = new ObjectNode(JsonNodeFactory.instance);
+ ArrayNode services = output.putArray("services");
+
+ jerseyService(services, "provision", "/provision/v1/", "/provision/application.wadl");
+ jerseyService(services, "statuspage", "/statuspage/v1/", "/statuspage/application.wadl");
+ jerseyService(services, "zone", "/zone/v1/", "/zone/application.wadl");
+ jerseyService(services, "zone", "/zone/v2/", "/zone/application.wadl");
+ jerseyService(services, "cost", "/cost/v1/", "/cost/application.wadl");
+ handlerService(services, "application", "/application/v4/");
+ handlerService(services, "deployment", "/deployment/v1/");
+ handlerService(services, "screwdriver", "/screwdriver/v1/release/vespa");
+
+ return output;
+ }
+
+ private void jerseyService(ArrayNode parent, String name, String url, String wadl) {
+ ObjectNode service = parent.addObject();
+ service.put("name", name);
+ service.put("url", controllerUri(url));
+ service.put("wadl", controllerUri(wadl));
+ }
+
+ private void handlerService(ArrayNode parent, String name, String url) {
+ ObjectNode service = parent.addObject();
+ service.put("name", name);
+ service.put("url", controllerUri(url));
+ }
+
+ private String controllerUri(String path) {
+ return uri.resolve(path).toString();
+ }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java
new file mode 100644
index 00000000000..81b07b81efb
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/SlimeJsonResponse.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A generic Json response using Slime for JSON encoding
+ *
+ * @author bratseth
+ */
+public class SlimeJsonResponse extends HttpResponse {
+
+ private final Slime slime;
+
+ public SlimeJsonResponse(Slime slime) {
+ super(200);
+ this.slime = slime;
+ }
+
+ public SlimeJsonResponse(int statusCode, Slime slime) {
+ super(statusCode);
+ this.slime = slime;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java
new file mode 100644
index 00000000000..1fc30b7d880
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/StringResponse.java
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.container.jdisc.HttpResponse;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author bratseth
+ */
+public class StringResponse extends HttpResponse {
+
+ private final String message;
+
+ public StringResponse(String message) {
+ super(200);
+ this.message = message;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ stream.write(message.getBytes("utf-8"));
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java
new file mode 100644
index 00000000000..479e7434f9b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/Uri.java
@@ -0,0 +1,64 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * A Uri which provides convenience methods for creating various manipulated copies.
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class Uri {
+
+ /** The URI instance wrapped by this */
+ private final URI uri;
+
+ public Uri(URI uri) {
+ this.uri = uri;
+ }
+
+ public Uri(String uri) {
+ try {
+ this.uri = new URI(uri);
+ }
+ catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Invalid URI", e);
+ }
+ }
+
+ /** Returns a uri with the given path appended and all parameters removed */
+ public Uri append(String pathElement) {
+ return new Uri(withoutParameters().withTrailingSlash() + pathElement);
+ }
+
+ public Uri withoutParameters() {
+ int parameterStart = uri.toString().indexOf("?");
+ if (parameterStart < 0)
+ return new Uri(uri.toString());
+ else
+ return new Uri(uri.toString().substring(0, parameterStart));
+ }
+
+ public Uri withPath(String path) {
+ try {
+ return new Uri(new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(),
+ uri.getPort(), path, uri.getQuery(), uri.getFragment()));
+ }
+ catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Could not add path '" + path + "' to " + this);
+ }
+ }
+
+ public Uri withTrailingSlash() {
+ if (toString().endsWith("/")) return this;
+ return new Uri(toString() + "/");
+ }
+
+ public URI toURI() { return uri; }
+
+ @Override
+ public String toString() { return uri.toString(); }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
new file mode 100644
index 00000000000..d701f3d57a0
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -0,0 +1,1065 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.google.common.base.Joiner;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.io.IOUtils;
+import com.yahoo.log.LogLevel;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.AlreadyExistsException;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.NotExistsException;
+import com.yahoo.vespa.hosted.controller.api.ActivateResult;
+import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.application.v4.ApplicationResource;
+import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
+import com.yahoo.vespa.hosted.controller.api.application.v4.TenantResource;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.CostJsonModelAdapter;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException;
+import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
+import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
+import com.yahoo.vespa.hosted.controller.restapi.Path;
+import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
+import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
+import com.yahoo.vespa.hosted.controller.restapi.StringResponse;
+import com.yahoo.vespa.hosted.controller.restapi.filter.SetBouncerPassthruHeaderFilter;
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+import com.yahoo.yolean.Exceptions;
+
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.ForbiddenException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.Principal;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+
+/**
+ * This implements the application/v4 API which is used to deploy and manage applications
+ * on hosted Vespa.
+ *
+ * @author bratseth
+ */
+public class ApplicationApiHandler extends LoggingRequestHandler {
+
+ private final Controller controller;
+ private final Authorizer authorizer;
+
+ public ApplicationApiHandler(Executor executor, AccessLog accessLog, Controller controller, Authorizer authorizer) {
+ super(executor, accessLog);
+ this.controller = controller;
+ this.authorizer = authorizer;
+ }
+
+ @Override
+ public Duration getTimeout() {
+ return Duration.ofMinutes(20); // deploys may take a long time;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ switch (request.getMethod()) {
+ case GET: return handleGET(request);
+ case PUT: return handlePUT(request);
+ case POST: return handlePOST(request);
+ case DELETE: return handleDELETE(request);
+ case OPTIONS: return handleOPTIONS(request);
+ default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
+ }
+ }
+ catch (ForbiddenException e) {
+ return ErrorResponse.forbidden(Exceptions.toMessageString(e));
+ }
+ catch (NotExistsException e) {
+ return ErrorResponse.notFoundError(Exceptions.toMessageString(e));
+ }
+ catch (IllegalArgumentException e) {
+ return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+ }
+ catch (ConfigServerException e) {
+ return ErrorResponse.from(e);
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+ return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+ }
+ }
+
+ private HttpResponse handleGET(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/application/v4/")) return root(request);
+ if (path.matches("/application/v4/user")) return authenticatedUser(request);
+ if (path.matches("/application/v4/tenant")) return tenants(request);
+ if (path.matches("/application/v4/tenant-pipeline")) return tenantPipelines();
+ if (path.matches("/application/v4/athensDomain")) return athensDomains(request);
+ if (path.matches("/application/v4/property")) return properties(request);
+ if (path.matches("/application/v4/cookiefreshness")) return cookieFreshness(request);
+ if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), path, request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/converge")) return waitForConvergence(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"))
+ return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handlePUT(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/application/v4/user")) return createUser(request);
+ if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"))
+ return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request);
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handlePOST(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/application/v4/tenant/{tenant}/migrateTenantToAthens")) return migrateTenant(path.get("tenant"), request);
+ if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"));
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploy(path.get("tenant"), path.get("application"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/log")) return log(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"));
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handleDELETE(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request);
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override"))
+ return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request);
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handleOPTIONS(HttpRequest request) {
+ // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother
+ // spelling out the methods supported at each path, which we should
+ EmptyJsonResponse response = new EmptyJsonResponse();
+ response.headers().put("Allow", "GET,PUT,POST,DELETE,OPTIONS");
+ return response;
+ }
+
+ private HttpResponse root(HttpRequest request) {
+ return new ResourceResponse(request,
+ "user", "tenant", "tenant-pipeline", "athensDomain", "property", "cookiefreshness");
+ }
+
+ private HttpResponse authenticatedUser(HttpRequest request) {
+ String userIdString = request.getProperty("userOverride");
+ if (userIdString == null)
+ userIdString = userFrom(request)
+ .orElseThrow(() -> new ForbiddenException("You must be authenticated or specify userOverride"));
+ UserId userId = new UserId(userIdString);
+
+ List<Tenant> tenants = controller.tenants().asList(userId);
+
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+ response.setString("user", userId.id());
+ Cursor tenantsArray = response.setArray("tenants");
+ for (Tenant tenant : tenants)
+ tenantInTenantsListToSlime(tenant, request.getUri(), tenantsArray.addObject());
+ response.setBool("tenantExists", tenants.stream().map(Tenant::getId).anyMatch(id -> id.isTenantFor(userId)));
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse tenants(HttpRequest request) {
+ Slime slime = new Slime();
+ Cursor response = slime.setArray();
+ for (Tenant tenant : controller.tenants().asList())
+ tenantInTenantsListToSlime(tenant, request.getUri(), response.addObject());
+ return new SlimeJsonResponse(slime);
+ }
+
+ /** Lists the screwdriver project id for each application */
+ private HttpResponse tenantPipelines() {
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+ Cursor pipelinesArray = response.setArray("tenantPipelines");
+ for (Application application : controller.applications().asList()) {
+ if ( ! application.deploymentJobs().projectId().isPresent()) continue;
+
+ Cursor pipelineObject = pipelinesArray.addObject();
+ pipelineObject.setString("screwdriverId", String.valueOf(application.deploymentJobs().projectId().get()));
+ pipelineObject.setString("tenant", application.id().tenant().value());
+ pipelineObject.setString("application", application.id().application().value());
+ pipelineObject.setString("instance", application.id().instance().value());
+ }
+ response.setArray("brokenTenantPipelines"); // not used but may need to be present
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse athensDomains(HttpRequest request) {
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+ Cursor array = response.setArray("data");
+ for (AthensDomain athensDomain : controller.getDomainList(request.getProperty("prefix"))) {
+ array.addString(athensDomain.id());
+ }
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse properties(HttpRequest request) {
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+ Cursor array = response.setArray("properties");
+ for (Map.Entry<PropertyId, Property> entry : controller.fetchPropertyList().entrySet()) {
+ Cursor propertyObject = array.addObject();
+ propertyObject.setString("propertyid", entry.getKey().id());
+ propertyObject.setString("property", entry.getValue().id());
+ }
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse cookieFreshness(HttpRequest request) {
+ Slime slime = new Slime();
+ String passThruHeader = request.getHeader(SetBouncerPassthruHeaderFilter.BOUNCER_PASSTHRU_HEADER_FIELD);
+ slime.setObject().setBool("shouldRefreshCookie",
+ ! SetBouncerPassthruHeaderFilter.BOUNCER_PASSTHRU_COOKIE_OK.equals(passThruHeader));
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse tenant(String tenantName, HttpRequest request) {
+ Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(tenantName));
+ if ( ! tenant.isPresent())
+ return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");
+ return new SlimeJsonResponse(toSlime(tenant.get(), request, true));
+ }
+
+ private HttpResponse applications(String tenantName, HttpRequest request) {
+ TenantName tenant = TenantName.from(tenantName);
+ Slime slime = new Slime();
+ Cursor array = slime.setArray();
+ for (Application application : controller.applications().asList(tenant))
+ toSlime(application, array.addObject(), request);
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse application(String tenantName, String applicationName, Path path, HttpRequest request) {
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+
+ com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default");
+ Application application =
+ controller.applications().get(applicationId)
+ .orElseThrow(() -> new NotExistsException(applicationId + " not found"));
+
+ // Currently deploying change
+ if (application.deploying().isPresent()) {
+ Cursor deployingObject = response.setObject("deploying");
+ if (application.deploying().get() instanceof Change.VersionChange)
+ deployingObject.setString("version", ((Change.VersionChange)application.deploying().get()).version().toString());
+ else if (((Change.ApplicationChange)application.deploying().get()).revision().isPresent())
+ toSlime(((Change.ApplicationChange)application.deploying().get()).revision().get(), deployingObject.setObject("revision"));
+ }
+
+ // Deployment jobs
+ Cursor deploymentsArray = response.setArray("deploymentJobs");
+ for (JobStatus job : application.deploymentJobs().jobStatus().values()) {
+ Cursor jobObject = deploymentsArray.addObject();
+ jobObject.setString("type", job.type().id());
+ jobObject.setBool("success", job.isSuccess());
+
+ job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered")));
+ job.lastCompleted().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastCompleted")));
+ job.firstFailing().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("firstFailing")));
+ job.lastSuccess().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastSuccess")));
+ }
+
+ // Compile version. The version that should be used when building an application
+ response.setString("compileVersion", application.compileVersion(controller).toFullString());
+
+ // Rotations
+ Cursor globalRotationsArray = response.setArray("globalRotations");
+ Set<URI> rotations = controller.getRotationUris(applicationId);
+ Map<String, RotationStatus> rotationHealthStatus =
+ rotations.isEmpty() ? Collections.emptyMap() : controller.getHealthStatus(rotations.iterator().next().getHost());
+ for (URI rotation : rotations)
+ globalRotationsArray.addString(rotation.toString());
+
+ // Deployments
+ Cursor instancesArray = response.setArray("instances");
+ for (Deployment deployment : application.deployments().values()) {
+ Cursor deploymentObject = instancesArray.addObject();
+ deploymentObject.setString("environment", deployment.zone().environment().value());
+ deploymentObject.setString("region", deployment.zone().region().value());
+ deploymentObject.setString("instance", application.id().instance().value()); // pointless
+ if ( ! rotations.isEmpty())
+ setRotationStatus(deployment, rotationHealthStatus, deploymentObject);
+ deploymentObject.setString("url", withPath(path.asString() +
+ "/environment/" + deployment.zone().environment().value() +
+ "/region/" + deployment.zone().region().value() +
+ "/instance/" + application.id().instance().value(),
+ request.getUri()).toString());
+ }
+
+ // Metrics
+ try {
+ MetricsService.ApplicationMetrics metrics = controller.metricsService().getApplicationMetrics(applicationId);
+ Cursor metricsObject = response.setObject("metrics");
+ metricsObject.setDouble("queryServiceQuality", metrics.queryServiceQuality());
+ metricsObject.setDouble("writeServiceQuality", metrics.writeServiceQuality());
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Failed getting Yamas metrics", e);
+ }
+
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse deployment(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+ ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
+ Application application = controller.applications().get(id)
+ .orElseThrow(() -> new NotExistsException(id + " not found"));
+
+ DeploymentId deploymentId = new DeploymentId(application.id(),
+ new Zone(Environment.from(environment), RegionName.from(region)));
+
+ Deployment deployment = application.deployments().get(deploymentId.zone());
+ if (deployment == null)
+ throw new NotExistsException(application + " is not deployed in " + deploymentId.zone());
+
+ Optional<InstanceEndpoints> deploymentEndpoints = controller.applications().getDeploymentEndpoints(deploymentId);
+
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+ Cursor serviceUrlArray = response.setArray("serviceUrls");
+ if (deploymentEndpoints.isPresent()) {
+ for (URI uri : deploymentEndpoints.get().getContainerEndpoints())
+ serviceUrlArray.addString(uri.toString());
+ }
+
+ response.setString("nodes", withPath("/zone/v2/" + environment + "/" + region + "/nodes/v2/node/?&recursive=true&application=" + tenantName + "." + applicationName + "." + instanceName, request.getUri()).toString());
+
+ Environment env = Environment.from(environment);
+ RegionName regionName = RegionName.from(region);
+ URI elkUrl = controller.getElkUri(env, regionName, deploymentId);
+ if (elkUrl != null)
+ response.setString("elkUrl", elkUrl.toString());
+
+ response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString());
+ response.setString("version", deployment.version().toFullString());
+ response.setString("revision", deployment.revision().id());
+ response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli());
+ Optional<Duration> deploymentTimeToLive = controller.zoneRegistry().getDeploymentTimeToLive(Environment.from(environment), RegionName.from(region));
+ deploymentTimeToLive.ifPresent(duration -> response.setLong("expiryTimeEpochMs", deployment.at().plus(duration).toEpochMilli()));
+
+ application.deploymentJobs().projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i)));
+ sourceRevisionToSlime(deployment.revision().source(), response);
+
+ com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, instanceName);
+ Zone zoneId = new Zone(Environment.from(environment), RegionName.from(region));
+
+ // Cost
+ try {
+ ApplicationCost appCost = controller.getApplicationCost(applicationId, zoneId);
+ Cursor costObject = response.setObject("cost");
+ CostJsonModelAdapter.toSlime(appCost, costObject);
+ } catch (NotFoundCheckedException nfce) {
+ log.log(Level.FINE, "Application cost data not found. " + nfce.getMessage());
+ }
+
+ // Metrics
+ try {
+ MetricsService.DeploymentMetrics metrics = controller.metricsService().getDeploymentMetrics(applicationId, zoneId);
+ Cursor metricsObject = response.setObject("metrics");
+ metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond());
+ metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond());
+ metricsObject.setDouble("documentCount", metrics.documentCount());
+ metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis());
+ metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis());
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Failed getting Yamas metrics", e);
+ }
+
+ return new SlimeJsonResponse(slime);
+ }
+
+ private void toSlime(ApplicationRevision revision, Cursor object) {
+ object.setString("hash", revision.id());
+ if (revision.source().isPresent())
+ sourceRevisionToSlime(revision.source(), object.setObject("source"));
+ }
+
+ private void sourceRevisionToSlime(Optional<SourceRevision> revision, Cursor object) {
+ if ( ! revision.isPresent()) return;
+ object.setString("gitRepository", revision.get().repository());
+ object.setString("gitBranch", revision.get().branch());
+ object.setString("gitCommit", revision.get().commit());
+ }
+
+ private URI monitoringSystemUri(DeploymentId deploymentId) {
+ return controller.zoneRegistry().getMonitoringSystemUri(deploymentId.zone().environment(),
+ deploymentId.zone().region(),
+ deploymentId.applicationId());
+ }
+
+ private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) {
+
+ // Check if request is authorized
+ Optional<Tenant> existingTenant = controller.tenants().tenant(new TenantId(tenantName));
+ if (!existingTenant.isPresent())
+ return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");
+
+ authorizer.throwIfUnauthorized(existingTenant.get().getId(), request);
+
+ // Decode payload (reason) and construct parameter to the configserver
+
+ Inspector requestData = toSlime(request.getData()).get();
+ String reason = mandatory("reason", requestData).asString();
+ String agent = authorizer.getUserId(request).toString();
+ long timestamp = controller.clock().instant().getEpochSecond();
+ EndpointStatus.Status status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out;
+ EndpointStatus endPointStatus = new EndpointStatus(status, reason, agent, timestamp);
+
+ // DeploymentId identifies the zone and application we are dealing with
+ DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
+ new Zone(Environment.from(environment), RegionName.from(region)));
+ try {
+ List<String> rotations = controller.applications().setGlobalRotationStatus(deploymentId, endPointStatus);
+ return new MessageResponse(String.format("Rotations %s successfully set to %s service", rotations.toString(), inService ? "in" : "out of"));
+ } catch (IOException e) {
+ return ErrorResponse.internalServerError("Unable to alter rotation status: " + e.getMessage());
+ }
+ }
+
+ private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) {
+
+ DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
+ new Zone(Environment.from(environment), RegionName.from(region)));
+
+ Slime slime = new Slime();
+ Cursor c1 = slime.setObject().setArray("globalrotationoverride");
+ try {
+ Map<String, EndpointStatus> rotations = controller.applications().getGlobalRotationStatus(deploymentId);
+ for (String rotation : rotations.keySet()) {
+ EndpointStatus currentStatus = rotations.get(rotation);
+ c1.addString(rotation);
+ Cursor c2 = c1.addObject();
+ c2.setString("status", currentStatus.getStatus().name());
+ c2.setString("reason", currentStatus.getReason() == null ? "" : currentStatus.getReason());
+ c2.setString("agent", currentStatus.getAgent() == null ? "" : currentStatus.getAgent());
+ c2.setLong("timestamp", currentStatus.getEpoch());
+ }
+ } catch (IOException e) {
+ return ErrorResponse.internalServerError("Unable to get rotation status: " + e.getMessage());
+ }
+
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse rotationStatus(String tenantName, String applicationName, String instanceName, String environment, String region) {
+
+ ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName);
+ Set<URI> rotations = controller.getRotationUris(applicationId);
+ if (rotations.isEmpty())
+ throw new NotExistsException("global rotation does not exist for '" + environment + "." + region + "'");
+
+ Slime slime = new Slime();
+ Cursor response = slime.setObject();
+
+ Map<String, RotationStatus> rotationHealthStatus = controller.getHealthStatus(rotations.iterator().next().getHost());
+
+ for (String rotationEndpoint : rotationHealthStatus.keySet()) {
+ if (rotationEndpoint.contains(toDns(environment)) && rotationEndpoint.contains(toDns(region))) {
+ Cursor bcpStatusObject = response.setObject("bcpStatus");
+ bcpStatusObject.setString("rotationStatus", rotationHealthStatus.getOrDefault(rotationEndpoint, RotationStatus.UNKNOWN).name());
+ }
+ }
+
+ return new SlimeJsonResponse(slime);
+ }
+
+ private HttpResponse waitForConvergence(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+ return new JacksonJsonResponse(controller.waitForConfigConvergence(new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
+ new Zone(Environment.from(environment), RegionName.from(region))),
+ asLong(request.getProperty("timeout"), 1000)));
+ }
+
+ private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+ ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region);
+ ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)),
+ new com.yahoo.config.provision.ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(),
+ controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)),
+ request.getUri());
+ response.setResponse(applicationView);
+ return response;
+ }
+
+ private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) {
+ Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath);
+ ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)),
+ new com.yahoo.config.provision.ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(),
+ controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)),
+ request.getUri());
+ response.setResponse(result, serviceName, restPath);
+ return response;
+ }
+
+ private HttpResponse createUser(HttpRequest request) {
+ Optional<String> username = userFrom(request);
+ if ( ! username.isPresent() ) throw new ForbiddenException("Not authenticated.");
+
+ try {
+ controller.tenants().createUserTenant(username.get());
+ return new MessageResponse("Created user '" + username.get() + "'");
+ } catch (AlreadyExistsException e) {
+ // Ok
+ return new MessageResponse("User '" + username + "' already exists");
+ }
+ }
+
+ private HttpResponse updateTenant(String tenantName, HttpRequest request) {
+ Optional<Tenant> existingTenant = controller.tenants().tenant(new TenantId(tenantName));
+ if ( ! existingTenant.isPresent()) return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");;
+
+ Inspector requestData = toSlime(request.getData()).get();
+
+ authorizer.throwIfUnauthorized(existingTenant.get().getId(), request);
+ Tenant updatedTenant;
+ switch (existingTenant.get().tenantType()) {
+ case USER: {
+ throw new BadRequestException("Cannot set property or OpsDB user group for user tenant");
+ }
+ case OPSDB: {
+ UserGroup userGroup = new UserGroup(mandatory("userGroup", requestData).asString());
+ updatedTenant = Tenant.createOpsDbTenant(new TenantId(tenantName),
+ userGroup,
+ new Property(mandatory("property", requestData).asString()),
+ optional("propertyId", requestData).map(PropertyId::new));
+ throwIfNotSuperUserOrPartOfOpsDbGroup(userGroup, request);
+ controller.tenants().updateTenant(updatedTenant, authorizer.getNToken(request));
+ break;
+ }
+ case ATHENS: {
+ if (requestData.field("userGroup").valid())
+ throw new BadRequestException("Cannot set OpsDB user group to Athens tenant");
+ updatedTenant = Tenant.createAthensTenant(new TenantId(tenantName),
+ new AthensDomain(mandatory("athensDomain", requestData).asString()),
+ new Property(mandatory("property", requestData).asString()),
+ optional("propertyId", requestData).map(PropertyId::new));
+ controller.tenants().updateTenant(updatedTenant, authorizer.getNToken(request));
+ break;
+ }
+ default: {
+ throw new BadRequestException("Unknown tenant type: " + existingTenant.get().tenantType());
+ }
+ }
+ return new SlimeJsonResponse(toSlime(updatedTenant, request, true));
+ }
+
+ private HttpResponse createTenant(String tenantName, HttpRequest request) {
+ if (new TenantId(tenantName).isUser())
+ return ErrorResponse.badRequest("Use User API to create user tenants.");
+
+ Inspector requestData = toSlime(request.getData()).get();
+
+ Tenant tenant = new Tenant(new TenantId(tenantName),
+ optional("userGroup", requestData).map(UserGroup::new),
+ optional("property", requestData).map(Property::new),
+ optional("athensDomain", requestData).map(AthensDomain::new),
+ optional("propertyId", requestData).map(PropertyId::new));
+ if (tenant.isOpsDbTenant())
+ throwIfNotSuperUserOrPartOfOpsDbGroup(new UserGroup(mandatory("userGroup", requestData).asString()), request);
+ if (tenant.isAthensTenant())
+ throwIfNotAthensDomainAdmin(new AthensDomain(mandatory("athensDomain", requestData).asString()), request);
+
+ controller.tenants().addTenant(tenant, authorizer.getNToken(request));
+ return new SlimeJsonResponse(toSlime(tenant, request, true));
+ }
+
+ private HttpResponse migrateTenant(String tenantName, HttpRequest request) {
+ TenantId tenantid = new TenantId(tenantName);
+ Inspector requestData = toSlime(request.getData()).get();
+ AthensDomain tenantDomain = new AthensDomain(mandatory("athensDomain", requestData).asString());
+ Property property = new Property(mandatory("property", requestData).asString());
+ PropertyId propertyId = new PropertyId(mandatory("propertyId", requestData).asString());
+
+ authorizer.throwIfUnauthorized(tenantid, request);
+ throwIfNotAthensDomainAdmin(tenantDomain, request);
+ NToken nToken = authorizer.getNToken(request)
+ .orElseThrow(() ->
+ new BadRequestException("The NToken for a domain admin is required to migrate tenant to Athens"));
+ Tenant tenant = controller.tenants().migrateTenantToAthens(tenantid, tenantDomain, propertyId, property, nToken);
+ return new SlimeJsonResponse(toSlime(tenant, request, true));
+ }
+
+ private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) {
+ authorizer.throwIfUnauthorized(new TenantId(tenantName), request);
+ Application application;
+ try {
+ application = controller.applications().createApplication(com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default"), authorizer.getNToken(request));
+ }
+ catch (ZmsException e) { // TODO: Push conversion down
+ if (e.getCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN)
+ throw new ForbiddenException("Not authorized to create application", e);
+ else
+ throw e;
+ }
+
+ Slime slime = new Slime();
+ toSlime(application, slime.setObject(), request);
+ return new SlimeJsonResponse(slime);
+ }
+
+ /** Trigger deployment of the last built application package, on a given version */
+ private HttpResponse deploy(String tenantName, String applicationName, HttpRequest request) {
+ ApplicationId id = ApplicationId.from(tenantName, applicationName, "default");
+ try (Lock lock = controller.applications().lock(id)) {
+ Application application = controller.applications().require(id);
+ if (application.deploying().isPresent())
+ throw new IllegalArgumentException("Can not start a deployment of " + application + " at this time: " +
+ application.deploying() + " is in progress");
+
+ Version version = decideDeployVersion(request);
+ if ( ! systemHasVersion(version))
+ throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " +
+ "Version is not active in this system. " +
+ "Active versions: " + controller.versionStatus().versions());
+
+ // Since we manually triggered it we don't want this to be self-triggering for the time being
+ controller.applications().store(application.with(application.deploymentJobs().asSelfTriggering(false)), lock);
+
+ controller.applications().deploymentTrigger().triggerChange(application.id(), new Change.VersionChange(version));
+ return new MessageResponse("Triggered deployment of " + application + " on version " + version);
+ }
+ }
+
+ private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+ DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
+ new Zone(Environment.from(environment), RegionName.from(region)));
+ // TODO: Propagate all filters
+ if (request.getProperty("hostname") != null)
+ controller.applications().restartHost(deploymentId, new Hostname(request.getProperty("hostname")));
+ else
+ controller.applications().restart(deploymentId);
+
+ // TODO: Change to return JSON
+ return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName,
+ ApplicationResource.API_PATH, applicationName,
+ EnvironmentResource.API_PATH, environment,
+ "region", region,
+ "instance", instanceName));
+ }
+
+ /**
+ * This returns and deletes recent error logs from this deployment, which is used by tenant deployment jobs to verify that
+ * the application is working. It is called for all production zones, also those in which the application is not present,
+ * and possibly before it is present, so failures are normal and expected.
+ */
+ private HttpResponse log(String tenantName, String applicationName, String instanceName, String environment, String region) {
+ try {
+ DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
+ new Zone(Environment.from(environment), RegionName.from(region)));
+ return new JacksonJsonResponse(controller.grabLog(deploymentId));
+ }
+ catch (RuntimeException e) {
+ Slime slime = new Slime();
+ slime.setObject();
+ return new SlimeJsonResponse(slime);
+ }
+ }
+
+ private HttpResponse deploy(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+ ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName);
+ Zone zone = new Zone(Environment.from(environment), RegionName.from(region));
+
+ Map<String, byte[]> dataParts = new MultipartParser().parse(request);
+ if ( ! dataParts.containsKey("deployOptions"))
+ return ErrorResponse.badRequest("Missing required form part 'deployOptions'");
+ if ( ! dataParts.containsKey("applicationZip"))
+ return ErrorResponse.badRequest("Missing required form part 'applicationZip'");
+
+ Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get();
+
+ DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.athens(), controller.zoneRegistry());
+ Tenant tenant = controller.tenants().tenant(new TenantId(tenantName)).orElseThrow(() -> new NotExistsException(new TenantId(tenantName)));
+ Principal principal = authorizer.getPrincipal(request);
+ if (principal instanceof AthensPrincipal) {
+ deployAuthorizer.throwIfUnauthorizedForDeploy(principal,
+ Environment.from(environment),
+ tenant,
+ applicationId);
+ } else { // In case of host-based principal
+ UserId userId = new UserId(principal.getName());
+ deployAuthorizer.throwIfUnauthorizedForDeploy(
+ Environment.from(environment),
+ userId,
+ tenant,
+ applicationId,
+ optional("screwdriverBuildJob", deployOptions).map(ScrewdriverId::new));
+ }
+
+
+ // TODO: get rid of the json object
+ DeployOptions deployOptionsJsonClass = new DeployOptions(screwdriverBuildJobFromSlime(deployOptions.field("screwdriverBuildJob")),
+ optional("vespaVersion", deployOptions).map(Version::new),
+ deployOptions.field("ignoreValidationErrors").asBool(),
+ deployOptions.field("deployCurrentVersion").asBool());
+ ActivateResult result = controller.applications().deployApplication(applicationId,
+ zone,
+ new ApplicationPackage(dataParts.get("applicationZip")),
+ deployOptionsJsonClass);
+ return new SlimeJsonResponse(toSlime(result, dataParts.get("applicationZip").length));
+ }
+
+ private HttpResponse deleteTenant(String tenantName, HttpRequest request) {
+ Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(tenantName));
+ if ( ! tenant.isPresent()) return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found"); // NOTE: The Jersey implementation would silently ignore this
+
+ authorizer.throwIfUnauthorized(new TenantId(tenantName), request);
+ controller.tenants().deleteTenant(new TenantId(tenantName), authorizer.getNToken(request));
+
+ // TODO: Change to a message response saying the tenant was deleted
+ return new SlimeJsonResponse(toSlime(tenant.get(), request, false));
+ }
+
+ private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) {
+ authorizer.throwIfUnauthorized(new TenantId(tenantName), request);
+
+ com.yahoo.config.provision.ApplicationId id = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, "default");
+ Application deleted = controller.applications().deleteApplication(id, authorizer.getNToken(request));
+ if (deleted == null)
+ return ErrorResponse.notFoundError("Could not delete application '" + id + "': Application not found");
+ return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead
+ }
+
+ private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region) {
+ Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName));
+
+ Zone zone = new Zone(Environment.from(environment), RegionName.from(region));
+ Deployment deployment = application.deployments().get(zone);
+ if (deployment == null)
+ return ErrorResponse.notFoundError("Could not deactivate: " + application + " is not deployed in " + zone);
+
+ controller.applications().deactivate(application, deployment, false);
+
+ // TODO: Change to return JSON
+ return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName,
+ ApplicationResource.API_PATH, applicationName,
+ EnvironmentResource.API_PATH, environment,
+ "region", region,
+ "instance", instanceName));
+ }
+
+ /**
+ * Promote application Chef environments. To be used by component jobs only
+ */
+ private HttpResponse promoteApplication(String tenantName, String applicationName) {
+ try{
+ ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system());
+ String sourceEnvironment = chefEnvironment.systemChefEnvironment();
+ String targetEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName));
+ controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment);
+ return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment));
+ } catch (Exception e) {
+ log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s)", tenantName, applicationName), e);
+ return ErrorResponse.internalServerError("Unable to promote Chef environments for application");
+ }
+ }
+
+ /**
+ * Promote application Chef environments for jobs that deploy applications
+ */
+ private HttpResponse promoteApplicationDeployment(String tenantName, String applicationName, String environmentName, String regionName) {
+ try {
+ ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system());
+ String sourceEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName));
+ String targetEnvironment = chefEnvironment.applicationTargetEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName), Environment.from(environmentName), RegionName.from(regionName));
+ controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment);
+ return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment));
+ } catch (Exception e) {
+ log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s %s.%s)", tenantName, applicationName, environmentName, regionName), e);
+ return ErrorResponse.internalServerError("Unable to promote Chef environments for application");
+ }
+ }
+
+ private Optional<String> userFrom(HttpRequest request) {
+ return authorizer.getPrincipalIfAny(request).map(Principal::getName);
+ }
+
+ private void toSlime(Tenant tenant, Cursor object, HttpRequest request, boolean listApplications) {
+ object.setString("type", tenant.tenantType().name());
+ tenant.getAthensDomain().ifPresent(a -> object.setString("athensDomain", a.id()));
+ tenant.getProperty().ifPresent(p -> object.setString("property", p.id()));
+ tenant.getPropertyId().ifPresent(p -> object.setString("propertyId", p.toString()));
+ tenant.getUserGroup().ifPresent(g -> object.setString("userGroup", g.id()));
+ Cursor applicationArray = object.setArray("applications");
+ if (listApplications) { // This cludge is needed because we call this after deleting the tenant. As this call makes another tenant lookup it will fail. TODO is to support lookup on tenant
+ for (Application application : controller.applications().asList(TenantName.from(tenant.getId().id()))) {
+ if (application.id().instance().isDefault()) // TODO: Skip non-default applications until supported properly
+ toSlime(application, applicationArray.addObject(), request);
+ }
+ }
+ }
+
+ // A tenant has different content when in a list ... antipattern, but not solvable before application/v5
+ private void tenantInTenantsListToSlime(Tenant tenant, URI requestURI, Cursor object) {
+ object.setString("tenant", tenant.getId().id());
+ Cursor metaData = object.setObject("metaData");
+ metaData.setString("type", tenant.tenantType().name());
+ tenant.getAthensDomain().ifPresent(a -> metaData.setString("athensDomain", a.id()));
+ tenant.getProperty().ifPresent(p -> metaData.setString("property", p.id()));
+ tenant.getUserGroup().ifPresent(g -> metaData.setString("userGroup", g.id()));
+ object.setString("url", withPath("/application/v4/tenant/" + tenant.getId().id(), requestURI).toString());
+ }
+
+ /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */
+ private URI withPath(String newPath, URI uri) {
+ try {
+ return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null);
+ }
+ catch (URISyntaxException e) {
+ throw new RuntimeException("Will not happen", e);
+ }
+ }
+
+ private void setRotationStatus(Deployment deployment, Map<String, RotationStatus> healthStatus, Cursor object) {
+ if ( ! deployment.zone().environment().equals(Environment.prod)) return;
+
+ Cursor bcpStatusObject = object.setObject("bcpStatus");
+ bcpStatusObject.setString("rotationStatus", findRotationStatus(deployment, healthStatus).name());
+ }
+
+ private RotationStatus findRotationStatus(Deployment deployment, Map<String, RotationStatus> healthStatus) {
+ for (String endpoint : healthStatus.keySet()) {
+ if (endpoint.contains(toDns(deployment.zone().environment().value())) &&
+ endpoint.contains(toDns(deployment.zone().region().value()))) {
+ return healthStatus.getOrDefault(endpoint, RotationStatus.UNKNOWN);
+ }
+ }
+
+ return RotationStatus.UNKNOWN;
+ }
+
+ private String toDns(String id) {
+ return id.replace('_', '-');
+ }
+
+ private long asLong(String valueOrNull, long defaultWhenNull) {
+ if (valueOrNull == null) return defaultWhenNull;
+ try {
+ return Long.parseLong(valueOrNull);
+ }
+ catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Expected an integer but got '" + valueOrNull + "'");
+ }
+ }
+
+ private void toSlime(JobStatus.JobRun jobRun, Cursor object) {
+ object.setString("version", jobRun.version().toFullString());
+ jobRun.revision().ifPresent(revision -> toSlime(revision, object.setObject("revision")));
+ object.setLong("at", jobRun.at().toEpochMilli());
+ }
+
+ private Slime toSlime(InputStream jsonStream) {
+ try {
+ byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000);
+ return SlimeUtils.jsonToSlime(jsonBytes);
+ } catch (IOException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ private void throwIfNotSuperUserOrPartOfOpsDbGroup(UserGroup userGroup, HttpRequest request) {
+ UserId userId = authorizer.getUserId(request);
+ if (!authorizer.isSuperUser(request) && !authorizer.isGroupMember(userId, userGroup) ) {
+ throw new ForbiddenException(String.format("User '%s' is not super user or part of the OpsDB user group '%s'",
+ userId.id(), userGroup.id()));
+ }
+ }
+
+ private void throwIfNotAthensDomainAdmin(AthensDomain tenantDomain, HttpRequest request) {
+ UserId userId = authorizer.getUserId(request);
+ if ( ! authorizer.isAthensDomainAdmin(userId, tenantDomain)) {
+ throw new ForbiddenException(
+ String.format("The user '%s' is not admin in Athens domain '%s'", userId.id(), tenantDomain.id()));
+ }
+ }
+
+ private Inspector mandatory(String key, Inspector object) {
+ if ( ! object.field(key).valid())
+ throw new IllegalArgumentException("'" + key + "' is missing");
+ return object.field(key);
+ }
+
+ private Optional<String> optional(String key, Inspector object) {
+ return SlimeUtils.optionalString(object.field(key));
+ }
+
+ private static String path(Object... elements) {
+ return Joiner.on("/").join(elements);
+ }
+
+ private Slime toSlime(Tenant tenant, HttpRequest request, boolean listApplications) {
+ Slime slime = new Slime();
+ toSlime(tenant, slime.setObject(), request, listApplications);
+ return slime;
+ }
+
+ private void toSlime(Application application, Cursor object, HttpRequest request) {
+ object.setString("application", application.id().application().value());
+ object.setString("instance", application.id().instance().value());
+ object.setString("url", withPath("/application/v4/tenant/" + application.id().tenant().value() +
+ "/application/" + application.id().application().value(), request.getUri()).toString());
+ }
+
+ private Slime toSlime(ActivateResult result, long applicationZipSizeBytes) {
+ Slime slime = new Slime();
+ Cursor object = slime.setObject();
+ object.setString("revisionId", result.getRevisionId().id());
+ object.setLong("applicationZipSize", applicationZipSizeBytes);
+ Cursor logArray = object.setArray("prepareMessages");
+ if (result.getPrepareResponse().log != null) {
+ for (Log logMessage : result.getPrepareResponse().log) {
+ Cursor logObject = logArray.addObject();
+ logObject.setLong("time", logMessage.time);
+ logObject.setString("level", logMessage.level);
+ logObject.setString("message", logMessage.message);
+ }
+ }
+
+ Cursor changeObject = object.setObject("configChangeActions");
+
+ Cursor restartActionsArray = changeObject.setArray("restart");
+ for (RestartAction restartAction : result.getPrepareResponse().configChangeActions.restartActions) {
+ Cursor restartActionObject = restartActionsArray.addObject();
+ restartActionObject.setString("clusterName", restartAction.clusterName);
+ restartActionObject.setString("clusterType", restartAction.clusterType);
+ restartActionObject.setString("serviceType", restartAction.serviceType);
+ serviceInfosToSlime(restartAction.services, restartActionObject.setArray("services"));
+ stringsToSlime(restartAction.messages, restartActionObject.setArray("messages"));
+ }
+
+ Cursor refeedActionsArray = changeObject.setArray("refeed");
+ for (RefeedAction refeedAction : result.getPrepareResponse().configChangeActions.refeedActions) {
+ Cursor refeedActionObject = refeedActionsArray.addObject();
+ refeedActionObject.setString("name", refeedAction.name);
+ refeedActionObject.setBool("allowed", refeedAction.allowed);
+ refeedActionObject.setString("documentType", refeedAction.documentType);
+ refeedActionObject.setString("clusterName", refeedAction.clusterName);
+ serviceInfosToSlime(refeedAction.services, refeedActionObject.setArray("services"));
+ stringsToSlime(refeedAction.messages, refeedActionObject.setArray("messages"));
+ }
+ return slime;
+ }
+
+ private void serviceInfosToSlime(List<ServiceInfo> serviceInfoList, Cursor array) {
+ for (ServiceInfo serviceInfo : serviceInfoList) {
+ Cursor serviceInfoObject = array.addObject();
+ serviceInfoObject.setString("serviceName", serviceInfo.serviceName);
+ serviceInfoObject.setString("serviceType", serviceInfo.serviceType);
+ serviceInfoObject.setString("configId", serviceInfo.configId);
+ serviceInfoObject.setString("hostName", serviceInfo.hostName);
+ }
+ }
+
+ private void stringsToSlime(List<String> strings, Cursor array) {
+ for (String string : strings)
+ array.addString(string);
+ }
+
+ // TODO: get rid of the json object
+ private Optional<ScrewdriverBuildJob> screwdriverBuildJobFromSlime(Inspector object) {
+ if ( ! object.valid() ) return Optional.empty();
+ Optional<ScrewdriverId> screwdriverId = optional("screwdriverId", object).map(ScrewdriverId::new);
+ return Optional.of(new ScrewdriverBuildJob(screwdriverId.orElse(null),
+ gitRevisionFromSlime(object.field("gitRevision"))));
+ }
+
+ // TODO: get rid of the json object
+ private GitRevision gitRevisionFromSlime(Inspector object) {
+ return new GitRevision(optional("repository", object).map(GitRepository::new).orElse(null),
+ optional("branch", object).map(GitBranch::new).orElse(null),
+ optional("commit", object).map(GitCommit::new).orElse(null));
+ }
+
+ private String readToString(InputStream stream) {
+ Scanner scanner = new Scanner(stream).useDelimiter("\\A");
+ if ( ! scanner.hasNext()) return null;
+ return scanner.next();
+ }
+
+ private boolean systemHasVersion(Version version) {
+ return controller.versionStatus().versions().stream().anyMatch(v -> v.versionNumber().equals(version));
+ }
+
+ private Version decideDeployVersion(HttpRequest request) {
+ String requestVersion = readToString(request.getData());
+ if (requestVersion != null)
+ return new Version(requestVersion);
+ else
+ return controller.systemVersion();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java
new file mode 100644
index 00000000000..7c32e48e218
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java
@@ -0,0 +1,43 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
+
+/**
+ * Represents Chef environments for applications/deployments. Used for promotion of Chef environments
+ *
+ * @author mortent
+ */
+public class ApplicationChefEnvironment {
+
+ private final String systemChefEnvironment;
+ private final String systemSuffix;
+
+ public ApplicationChefEnvironment(SystemName system) {
+ if (system == SystemName.main) {
+ systemChefEnvironment = "hosted-verified-prod";
+ systemSuffix = "";
+ } else {
+ systemChefEnvironment = "hosted-infra-cd";
+ systemSuffix = "-cd";
+ }
+ }
+
+ public String systemChefEnvironment() {
+ return systemChefEnvironment;
+ }
+
+ public String applicationSourceEnvironment(TenantName tenantName, ApplicationName applicationName) {
+ // placeholder and component already used in legacy chef promotion
+ return String.format("hosted-instance%s_%s_%s_placeholder_component_default", systemSuffix, tenantName, applicationName);
+ }
+
+ public String applicationTargetEnvironment(TenantName tenantName, ApplicationName applicationName, Environment environment, RegionName regionName) {
+ return String.format("hosted-instance%s_%s_%s_%s_%s_default", systemSuffix, tenantName, applicationName, regionName, environment);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java
new file mode 100644
index 00000000000..8dff39779b9
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java
@@ -0,0 +1,164 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.common.ContextAttributes;
+import com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter;
+import com.yahoo.vespa.hosted.controller.restapi.filter.UnauthenticatedUserPrincipal;
+
+import javax.ws.rs.ForbiddenException;
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.SecurityContext;
+import java.security.Principal;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+
+
+/**
+ * @author Stian Kristoffersen
+ * @author Tony Vaagenes
+ * @author bjorncs
+ */
+// TODO: Make this an interface
+public class Authorizer {
+
+ private static final Logger log = Logger.getLogger(Authorizer.class.getName());
+
+ // Must be kept in sync with bouncer filter configuration.
+ private static final String VESPA_HOSTED_ADMIN_ROLE = "10707.A";
+
+ private static final Set<UserId> SCREWDRIVER_USERS = ImmutableSet.of(new UserId("screwdrv"),
+ new UserId("screwdriver"),
+ new UserId("sdrvtest"),
+ new UserId("screwdriver-test"));
+
+ private final Controller controller;
+ private final ZmsClientFactory zmsClientFactory;
+ private final EntityService entityService;
+ private final Athens athens;
+
+ public Authorizer(Controller controller, EntityService entityService) {
+ this.controller = controller;
+ this.zmsClientFactory = controller.athens().zmsClientFactory();
+ this.entityService = entityService;
+ this.athens = controller.athens();
+ }
+
+ public void throwIfUnauthorized(TenantId tenantId, HttpRequest request) throws ForbiddenException {
+ if (isReadOnlyMethod(request.getMethod().name())) return;
+ if (isSuperUser(request)) return;
+
+ Optional<Tenant> tenant = controller.tenants().tenant(tenantId);
+ if ( ! tenant.isPresent()) return;
+
+ UserId userId = getUserId(request);
+ if (isTenantAdmin(userId, tenant.get())) return;
+
+ throw loggedForbiddenException("User " + userId + " does not have write access to tenant " + tenantId);
+ }
+
+ public UserId getUserId(HttpRequest request) {
+ String name = getPrincipal(request).getName();
+ if (name == null)
+ throw loggedForbiddenException("Not authorized: User name is null");
+ return new UserId(name);
+ }
+
+ /** Returns the principal or throws forbidden */ // TODO: Avoid REST exceptions
+ public Principal getPrincipal(HttpRequest request) {
+ return getPrincipalIfAny(request).orElseThrow(() -> Authorizer.loggedForbiddenException("User is not authenticated"));
+ }
+
+ /** Returns the principal if there is any */
+ public Optional<Principal> getPrincipalIfAny(HttpRequest request) {
+ return securityContextOf(request).map(SecurityContext::getUserPrincipal);
+ }
+
+ public Optional<NToken> getNToken(HttpRequest request) {
+ String nTokenHeader = (String)request.getJDiscRequest().context().get(NTokenRequestFilter.NTOKEN_HEADER);
+ return Optional.ofNullable(nTokenHeader).map(athens::nTokenFrom);
+ }
+
+ public boolean isSuperUser(HttpRequest request) {
+ // TODO Check membership of admin role in Vespa's Athens domain
+ return isMemberOfVespaBouncerGroup(request) || isScrewdriverPrincipal(athens, getPrincipal(request));
+ }
+
+ public static boolean isScrewdriverPrincipal(Athens athens, Principal principal) {
+ if (principal instanceof UnauthenticatedUserPrincipal) // Host-based authentication
+ return SCREWDRIVER_USERS.contains(new UserId(principal.getName()));
+ else if (principal instanceof AthensPrincipal)
+ return ((AthensPrincipal)principal).getDomain().equals(athens.screwdriverDomain());
+ else
+ return false;
+ }
+
+ private static ForbiddenException loggedForbiddenException(String message, Object... args) {
+ String formattedMessage = String.format(message, args);
+ log.info(formattedMessage);
+ return new ForbiddenException(formattedMessage);
+ }
+
+ private boolean isTenantAdmin(UserId userId, Tenant tenant) {
+ switch (tenant.tenantType()) {
+ case ATHENS:
+ return isAthensTenantAdmin(userId, tenant.getAthensDomain().get());
+ case OPSDB:
+ return isGroupMember(userId, tenant.getUserGroup().get());
+ case USER:
+ return isUserTenantOwner(tenant.getId(), userId);
+ }
+ throw new IllegalArgumentException("Unknown tenant type: " + tenant.tenantType());
+ }
+
+ private boolean isAthensTenantAdmin(UserId userId, AthensDomain tenantDomain) {
+ return zmsClientFactory.createClientWithServicePrincipal()
+ .hasTenantAdminAccess(athens.principalFrom(userId), tenantDomain);
+ }
+
+ public boolean isAthensDomainAdmin(UserId userId, AthensDomain tenantDomain) {
+ return zmsClientFactory.createClientWithServicePrincipal()
+ .isDomainAdmin(athens.principalFrom(userId), tenantDomain);
+ }
+
+ public boolean isGroupMember(UserId userId, UserGroup userGroup) {
+ return entityService.isGroupMember(userId, userGroup);
+ }
+
+ private static boolean isUserTenantOwner(TenantId tenantId, UserId userId) {
+ return tenantId.equals(userId.toTenantId());
+ }
+
+ public static boolean environmentRequiresAuthorization(Environment environment) {
+ return environment != Environment.dev && environment != Environment.perf;
+ }
+
+ private static boolean isReadOnlyMethod(String method) {
+ return method.equals(HttpMethod.GET) || method.equals(HttpMethod.HEAD) || method.equals(HttpMethod.OPTIONS);
+ }
+
+ private boolean isMemberOfVespaBouncerGroup(HttpRequest request) {
+ Optional<SecurityContext> securityContext = securityContextOf(request);
+ if ( ! securityContext.isPresent() ) throw Authorizer.loggedForbiddenException("User is not authenticated");
+ return securityContext.get().isUserInRole(Authorizer.VESPA_HOSTED_ADMIN_ROLE);
+ }
+
+ protected Optional<SecurityContext> securityContextOf(HttpRequest request) {
+ return Optional.ofNullable((SecurityContext)request.getJDiscRequest().context().get(ContextAttributes.SECURITY_CONTEXT_ATTRIBUTE));
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java
new file mode 100644
index 00000000000..5c7cdfdae0a
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java
@@ -0,0 +1,117 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.restapi.filter.UnauthenticatedUserPrincipal;
+
+import javax.ws.rs.ForbiddenException;
+import java.security.Principal;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.hosted.controller.restapi.application.Authorizer.environmentRequiresAuthorization;
+import static com.yahoo.vespa.hosted.controller.restapi.application.Authorizer.isScrewdriverPrincipal;
+
+/**
+ * @author bjorncs
+ * @author gjoranv
+ */
+public class DeployAuthorizer {
+
+ private static final Logger log = Logger.getLogger(DeployAuthorizer.class.getName());
+
+ private final Athens athens;
+ private final ZoneRegistry zoneRegistry;
+
+ public DeployAuthorizer(Athens athens, ZoneRegistry zoneRegistry) {
+ this.athens = athens;
+ this.zoneRegistry = zoneRegistry;
+ }
+
+ public void throwIfUnauthorizedForDeploy(Principal principal,
+ Environment environment,
+ Tenant tenant,
+ ApplicationId applicationId) {
+ if (athensCredentialsRequired(environment, tenant, applicationId, principal))
+ checkAthensCredentials(principal, tenant, applicationId);
+ }
+
+ // TODO: inline when deployment via ssh is removed
+ private boolean athensCredentialsRequired(Environment environment, Tenant tenant, ApplicationId applicationId, Principal principal) {
+ if (!environmentRequiresAuthorization(environment)) return false;
+
+ if (! isScrewdriverPrincipal(athens, principal))
+ throw loggedForbiddenException(
+ "Principal '%s' is not a screwdriver principal, and does not have deploy access to application '%s'",
+ principal.getName(), applicationId.toShortString());
+
+ return tenant.isAthensTenant();
+ }
+
+
+ // TODO: inline when deployment via ssh is removed
+ private void checkAthensCredentials(Principal principal, Tenant tenant, ApplicationId applicationId) {
+ AthensDomain domain = tenant.getAthensDomain().get();
+ if (! (principal instanceof AthensPrincipal))
+ throw loggedForbiddenException("Principal '%s' is not authenticated.", principal.getName());
+
+ AthensPrincipal athensPrincipal = (AthensPrincipal)principal;
+ if ( ! hasDeployAccessToAthensApplication(athensPrincipal, domain, applicationId))
+ throw loggedForbiddenException(
+ "Screwdriver principal '%1$s' does not have deploy access to '%2$s'. " +
+ "Either the application has not been created at " + zoneRegistry.getDashboardUri() + " or " +
+ "'%1$s' is not added to the application's deployer role in Athens domain '%3$s'.",
+ athensPrincipal, applicationId, tenant.getAthensDomain().get());
+ }
+
+ private static ForbiddenException loggedForbiddenException(String message, Object... args) {
+ String formattedMessage = String.format(message, args);
+ log.info(formattedMessage);
+ return new ForbiddenException(formattedMessage);
+ }
+
+ /**
+ * @deprecated Only usable for ssh. Use the method that takes Principal instead of UserId and screwdriverId.
+ */
+ @Deprecated
+ public void throwIfUnauthorizedForDeploy(Environment environment,
+ UserId userId,
+ Tenant tenant,
+ ApplicationId applicationId,
+ Optional<ScrewdriverId> optionalScrewdriverId) {
+
+ Principal principal = new UnauthenticatedUserPrincipal(userId.id());
+
+ if (athensCredentialsRequired(environment, tenant, applicationId, principal)) {
+ ScrewdriverId screwdriverId = optionalScrewdriverId.orElseThrow(
+ () -> loggedForbiddenException("Screwdriver id must be provided when deploying from Screwdriver."));
+ principal = athens.principalFrom(screwdriverId);
+ checkAthensCredentials(principal, tenant, applicationId);
+ }
+ }
+
+ private boolean hasDeployAccessToAthensApplication(AthensPrincipal principal, AthensDomain domain, ApplicationId applicationId) {
+ try {
+ return athens.zmsClientFactory().createClientWithServicePrincipal()
+ .hasApplicationAccess(
+ principal,
+ ApplicationAction.deploy,
+ domain,
+ new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value()));
+ } catch (ZmsException e) {
+ throw loggedForbiddenException(
+ "Failed to authorize deployment through Athens. If this problem persists, " +
+ "please create ticket at yo/vespa-support. (" + e.getMessage() + ")");
+ }
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java
new file mode 100644
index 00000000000..3e8d4182c42
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java
@@ -0,0 +1,25 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author bratseth
+ */
+public class EmptyJsonResponse extends HttpResponse {
+
+ public EmptyJsonResponse() {
+ super(200);
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException { }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java
new file mode 100644
index 00000000000..cfd6feccf01
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JacksonJsonResponse.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.container.jdisc.HttpResponse;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author bratseth
+ */
+public class JacksonJsonResponse extends HttpResponse {
+
+ private final JsonNode node;
+
+ public JacksonJsonResponse(JsonNode node) {
+ super(200);
+ this.node = node;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new ObjectMapper().writeValue(stream, node);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java
new file mode 100644
index 00000000000..75f4ff68f1e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java
@@ -0,0 +1,72 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import org.apache.commons.fileupload.MultipartStream;
+import org.apache.commons.fileupload.ParameterParser;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Provides reading a multipart/form-data request type into a map of bytes for each part,
+ * indexed by the parts (form field) name.
+ *
+ * @author bratseth
+ */
+public class MultipartParser {
+
+ /**
+ * Parses the given multi-part request and returns all the parts indexed by their name.
+ *
+ * @throws IllegalArgumentException if this request is not a well-formed request with Content-Type multipart/form-data
+ */
+ public Map<String, byte[]> parse(HttpRequest request) {
+ try {
+ ParameterParser parameterParser = new ParameterParser();
+ Map<String, String> contentType = parameterParser.parse(request.getHeader("Content-Type"), ';');
+ if ( ! contentType.containsKey("multipart/form-data"))
+ throw new IllegalArgumentException("Expected a multipart message, but got Content-Type: " +
+ request.getHeader("Content-Type"));
+ String boundary = contentType.get("boundary");
+ if (boundary == null)
+ throw new IllegalArgumentException("Missing boundary property in Content-Type header");
+ MultipartStream multipartStream = new MultipartStream(request.getData(), boundary.getBytes(),
+ 1000 * 1000,
+ null);
+ boolean nextPart = multipartStream.skipPreamble();
+ Map<String, byte[]> parts = new HashMap<>();
+ while (nextPart) {
+ String[] headers = multipartStream.readHeaders().split("\r\n");
+ String contentDispositionContent = findContentDispositionHeader(headers);
+ if (contentDispositionContent == null)
+ throw new IllegalArgumentException("Missing Content-Disposition header in a multipart body part");
+ Map<String, String> contentDisposition = parameterParser.parse(contentDispositionContent, ';');
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ multipartStream.readBodyData(output);
+ parts.put(contentDisposition.get("name"), output.toByteArray());
+ nextPart = multipartStream.readBoundary();
+ }
+ return parts;
+ }
+ catch(MultipartStream.MalformedStreamException e) {
+ throw new IllegalArgumentException("Malformed multipart/form-data request", e);
+ }
+ catch(IOException e) {
+ throw new IllegalArgumentException("IO error reading multipart request " + request.getUri(), e);
+ }
+ }
+
+ private String findContentDispositionHeader(String[] headers) {
+ String contentDisposition = "Content-Disposition:";
+ for (String header : headers) {
+ if (header.length() < contentDisposition.length()) continue;
+ if ( ! header.substring(0, contentDisposition.length()).equalsIgnoreCase(contentDisposition)) continue;
+ return header.substring(contentDisposition.length() + 1);
+ }
+ return null;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java
new file mode 100644
index 00000000000..6a448e475c5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java
@@ -0,0 +1,191 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.restapi.Uri;
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+import com.yahoo.vespa.serviceview.bindings.ClusterView;
+import com.yahoo.vespa.serviceview.bindings.ServiceView;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A response containing a service view for an application deployment.
+ * This does not define the API response but merely proxies the API response provided by Vespa, with URLs
+ * rewritten to include zone and application information allow proxying through the controller
+ *
+ * @author Steinar Knutsen
+ * @author bratseth
+ */
+class ServiceApiResponse extends HttpResponse {
+
+ private final Zone zone;
+ private final ApplicationId application;
+ private final List<URI> configServerURIs;
+ private final Slime slime;
+ private final Uri requestUri;
+
+ // Only set for one of the setResponse calls
+ private String serviceName = null;
+ private String restPath = null;
+
+ public ServiceApiResponse(Zone zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) {
+ super(200);
+ this.zone = zone;
+ this.application = application;
+ this.configServerURIs = configServerURIs;
+ this.slime = new Slime();
+ this.requestUri = new Uri(requestUri).withoutParameters();
+ }
+
+ public void setResponse(ApplicationView applicationView) {
+ Cursor clustersArray = slime.setObject().setArray("clusters");
+ for (ClusterView clusterView : applicationView.clusters) {
+ Cursor clusterObject = clustersArray.addObject();
+ clusterObject.setString("name", clusterView.name);
+ clusterObject.setString("type", clusterView.type);
+ setNullableString("url", rewriteIfUrl(clusterView.url, requestUri), clusterObject);
+ Cursor servicesArray = clusterObject.setArray("services");
+ for (ServiceView serviceView : clusterView.services) {
+ Cursor serviceObject = servicesArray.addObject();
+ setNullableString("url", rewriteIfUrl(serviceView.url, requestUri), serviceObject);
+ serviceObject.setString("serviceType", serviceView.serviceType);
+ serviceObject.setString("serviceName", serviceView.serviceName);
+ serviceObject.setString("configId", serviceView.configId);
+ serviceObject.setString("host", serviceView.host);
+ }
+ }
+ }
+
+ public void setResponse(Map<?,?> responseData, String serviceName, String restPath) {
+ this.serviceName = serviceName;
+ this.restPath = restPath;
+ mapToSlime(responseData, slime.setObject());
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void mapToSlime(Map<?,?> data, Cursor object) {
+ for (Map.Entry<String, Object> entry : ((Map<String, Object>)data).entrySet())
+ fieldToSlime(entry.getKey(), entry.getValue(), object);
+ }
+
+ private void fieldToSlime(String key, Object value, Cursor object) {
+ if (value instanceof String) {
+ if (key.equals("url") || key.equals("link"))
+ value = rewriteIfUrl((String)value, generateLocalLinkPrefix(serviceName, restPath));
+ setNullableString(key, (String)value, object);
+ }
+ else if (value instanceof Integer) {
+ object.setLong(key, (int)value);
+ }
+ else if (value instanceof Long) {
+ object.setLong(key, (long)value);
+ }
+ else if (value instanceof Float) {
+ object.setDouble(key, (double)value);
+ }
+ else if (value instanceof Double) {
+ object.setDouble(key, (double)value);
+ }
+ else if (value instanceof List) {
+ listToSlime((List)value, object.setArray(key));
+ }
+ else if (value instanceof Map) {
+ mapToSlime((Map<?,?>)value, object.setObject(key));
+ }
+ }
+
+ private void listToSlime(List<?> list, Cursor array) {
+ for (Object entry : list)
+ entryToSlime(entry, array);
+ }
+
+ private void entryToSlime(Object entry, Cursor array) {
+ if (entry instanceof String)
+ addNullableString(rewriteIfUrl((String)entry, generateLocalLinkPrefix(serviceName, restPath)), array);
+ else if (entry instanceof Integer)
+ array.addLong((long)entry);
+ else if (entry instanceof Long)
+ array.addLong((long)entry);
+ else if (entry instanceof Float)
+ array.addDouble((double)entry);
+ else if (entry instanceof Double)
+ array.addDouble((double)entry);
+ else if (entry instanceof List)
+ listToSlime((List)entry, array.addArray());
+ else if (entry instanceof Map)
+ mapToSlime((Map)entry, array.addObject());
+ }
+
+ private String rewriteIfUrl(String urlOrAnyString, Uri requestUri) {
+ if (urlOrAnyString == null) return null;
+
+ String hostPattern = "(" +
+ String.join(
+ "|", configServerURIs.stream()
+ .map(URI::toString)
+ .map(s -> s.substring(0, s.length() -1))
+ .map(Pattern::quote)
+ .toArray(String[]::new))
+ + ")";
+
+ String remoteServicePath = "/serviceview/"
+ + "v1/tenant/" + application.tenant().value()
+ + "/application/" + application.application().value()
+ + "/environment/" + zone.environment().value()
+ + "/region/" + zone.region().value()
+ + "/instance/" + application.instance()
+ + "/service/";
+
+ Pattern remoteServiceResourcePattern = Pattern.compile("^(" + hostPattern + Pattern.quote(remoteServicePath) + ")");
+ Matcher matcher = remoteServiceResourcePattern.matcher(urlOrAnyString);
+
+ if (matcher.find()) {
+ String proxiedPath = urlOrAnyString.substring(matcher.group().length());
+ return requestUri.append(proxiedPath).toString();
+ } else {
+ return urlOrAnyString; // not a service url
+ }
+ }
+
+ private Uri generateLocalLinkPrefix(String identifier, String restPath) {
+ String proxiedPath = identifier + "/" + restPath;
+
+ if (this.requestUri.toString().endsWith(proxiedPath)) {
+ return new Uri(this.requestUri.toString().substring(0, this.requestUri.toString().length() - proxiedPath.length()));
+ } else {
+ throw new IllegalStateException("Expected the resource path '" + this.requestUri + "' to end with '" + proxiedPath + "'");
+ }
+ }
+
+ private void setNullableString(String key, String valueOrNull, Cursor receivingObject) {
+ if (valueOrNull == null)
+ receivingObject.setNix(key);
+ else
+ receivingObject.setString(key, valueOrNull);
+ }
+
+ private void addNullableString(String valueOrNull, Cursor receivingArray) {
+ if (valueOrNull == null)
+ receivingArray.addNix();
+ else
+ receivingArray.addString(valueOrNull);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
new file mode 100644
index 00000000000..e02a31440ce
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -0,0 +1,84 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance;
+import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
+import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
+import com.yahoo.vespa.hosted.controller.restapi.Path;
+import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
+import com.yahoo.yolean.Exceptions;
+
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+
+/**
+ * This implements the controller/v1 API which provides operators with information about,
+ * and control over the Controller.
+ *
+ * @author bratseth
+ */
+public class ControllerApiHandler extends LoggingRequestHandler {
+
+ private final ControllerMaintenance maintenance;
+
+ public ControllerApiHandler(Executor executor, AccessLog accessLog, ControllerMaintenance maintenance) {
+ super(executor, accessLog);
+ this.maintenance = maintenance;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ switch (request.getMethod()) {
+ case GET: return handleGET(request);
+ case POST: return handlePOST(request);
+ case DELETE: return handleDELETE(request);
+ default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
+ }
+ }
+ catch (IllegalArgumentException e) {
+ return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+ return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+ }
+ }
+
+ private HttpResponse handleGET(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/controller/v1/")) return root(request);
+ if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl());
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handlePOST(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/controller/v1/maintenance/inactive/{jobName}"))
+ return setActive(path.get("jobName"), false);
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handleDELETE(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/controller/v1/maintenance/inactive/{jobName}"))
+ return setActive(path.get("jobName"), true);
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse root(HttpRequest request) {
+ return new ResourceResponse(request, "maintenance");
+ }
+
+ private HttpResponse setActive(String jobName, boolean active) {
+ if ( ! maintenance.jobControl().jobs().contains(jobName))
+ return ErrorResponse.notFoundError("No job named '" + jobName + "'");
+ maintenance.jobControl().setActive(jobName, active);
+ return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java
new file mode 100644
index 00000000000..e7d1b3e0ed8
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/JobsResponse.java
@@ -0,0 +1,46 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A response containing maintenance job status
+ *
+ * @author bratseth
+ */
+public class JobsResponse extends HttpResponse {
+
+ private final JobControl jobControl;
+
+ public JobsResponse(JobControl jobControl) {
+ super(200);
+ this.jobControl = jobControl;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ Slime slime = new Slime();
+ Cursor root = slime.setObject();
+
+ Cursor jobArray = root.setArray("jobs");
+ for (String jobName : jobControl.jobs())
+ jobArray.addObject().setString("name", jobName);
+
+ Cursor inactiveArray = root.setArray("inactive");
+ for (String jobName : jobControl.inactiveJobs())
+ inactiveArray.addString(jobName);
+
+ new JsonFormat(true).encode(stream, slime);
+ }
+
+ @Override
+ public String getContentType() { return "application/json"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
new file mode 100644
index 00000000000..affd679f2c2
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -0,0 +1,122 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.deployment;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
+import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
+import com.yahoo.vespa.hosted.controller.restapi.Uri;
+import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse;
+import com.yahoo.vespa.hosted.controller.restapi.Path;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Instant;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+
+/**
+ * This implements the deployment/v1 API which provides information about the status of Vespa platform and
+ * application deployments.
+ *
+ * @author bratseth
+ */
+public class DeploymentApiHandler extends LoggingRequestHandler {
+
+ private final Controller controller;
+
+ public DeploymentApiHandler(Executor executor, AccessLog accessLog, Controller controller) {
+ super(executor, accessLog);
+ this.controller = controller;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ switch (request.getMethod()) {
+ case GET: return handleGET(request);
+ case OPTIONS: return handleOPTIONS();
+ default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
+ }
+ }
+ catch (IllegalArgumentException e) {
+ return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+ }
+ catch (RuntimeException e) {
+ log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+ return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+ }
+ }
+
+ private HttpResponse handleGET(HttpRequest request) {
+ Path path = new Path(request.getUri().getPath());
+ if (path.matches("/deployment/v1/")) return root(request);
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
+
+ private HttpResponse handleOPTIONS() {
+ // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother
+ // spelling out the methods supported at each path, which we should
+ EmptyJsonResponse response = new EmptyJsonResponse();
+ response.headers().put("Allow", "GET,OPTIONS");
+ return response;
+ }
+
+ private HttpResponse root(HttpRequest request) {
+ Slime slime = new Slime();
+ Cursor root = slime.setObject();
+ Cursor platformArray = root.setArray("versions");
+ for (VespaVersion version : controller.versionStatus().versions()) {
+ Cursor versionObject = platformArray.addObject();
+ versionObject.setString("version", version.versionNumber().toString());
+ versionObject.setString("confidence", version.confidence().name());
+ versionObject.setString("commit", version.releaseCommit());
+ versionObject.setLong("date", version.releasedAt().toEpochMilli());
+ versionObject.setBool("controllerVersion", version.isSelfVersion());
+ versionObject.setBool("systemVersion", version.isCurrentSystemVersion());
+
+ Cursor configServerArray = versionObject.setArray("configServers");
+ for (String configServerHostnames : version.configServerHostnames()) {
+ Cursor configServerObject = configServerArray.addObject();
+ configServerObject.setString("hostname", configServerHostnames);
+ }
+
+ Cursor failingArray = versionObject.setArray("failingApplications");
+ for (ApplicationId id : version.statistics().failing()) {
+ Optional<Application> application = controller.applications().get(id);
+ if ( ! application.isPresent()) continue; // deleted just now
+
+ Instant failingSince = application.get().deploymentJobs().failingSince();
+ if (failingSince == null) continue; // started working just now
+
+ Cursor applicationObject = failingArray.addObject();
+ toSlime(id, applicationObject, request);
+ applicationObject.setLong("failingSince", failingSince.toEpochMilli());
+ }
+
+ Cursor productionArray = versionObject.setArray("productionApplications");
+ for (ApplicationId id : version.statistics().production())
+ toSlime(id, productionArray.addObject(), request);
+ }
+ return new SlimeJsonResponse(slime);
+ }
+
+ private void toSlime(ApplicationId id, Cursor object, HttpRequest request) {
+ object.setString("tenant", id.tenant().value());
+ object.setString("application", id.application().value());
+ object.setString("instance", id.instance().value());
+ object.setString("url", new Uri(request.getUri()).withPath("/application/v4" +
+ "/tenant/" + id.tenant().value() +
+ "/application/" + id.application().value())
+ .toString());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java
new file mode 100644
index 00000000000..aea59c16cd5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlHeaders.java
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+
+/**
+ * @author gv
+ */
+public interface AccessControlHeaders {
+
+ String CORS_PREFLIGHT_REQUEST_CACHE_TTL = Long.toString(DAYS.toSeconds(7));
+
+ String ALLOW_ORIGIN_HEADER = "Access-Control-Allow-Origin";
+
+ Map<String, String> ACCESS_CONTROL_HEADERS = ImmutableMap.of(
+ "Access-Control-Max-Age", CORS_PREFLIGHT_REQUEST_CACHE_TTL,
+ "Access-Control-Allow-Headers", "Origin,Content-Type,Accept,Yahoo-Principal-Auth",
+ "Access-Control-Allow-Methods", "OPTIONS,GET,PUT,DELETE,POST",
+ "Access-Control-Allow-Credentials", "true"
+ );
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java
new file mode 100644
index 00000000000..8dace5d56dc
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilter.java
@@ -0,0 +1,68 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.google.inject.Inject;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.ContentChannel;
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.HttpResponse;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig;
+import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Before;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS;
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS;
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER;
+
+/**
+ * <p>
+ * This filter makes sure we respond as quickly as possible to CORS pre-flight requests
+ * which browsers transmit before the Hosted Vespa dashboard code is allowed to send a "real" request.
+ * </p>
+ * <p>
+ * An "Access-Control-Max-Age" header is added so that the browser will cache the result of this pre-flight request,
+ * further improving the responsiveness of the Hosted Vespa dashboard application.
+ * </p>
+ * <p>
+ * Runs after all standard security request filters, but before BouncerFilter, as the browser does not send
+ * credentials with pre-flight requests.
+ * </p>
+ *
+ * @author andreer
+ * @author gv
+ */
+@After({"InputValidationFilter","RemoteIPFilter", "DoNotTrackRequestFilter", "CookieDataRequestFilter"})
+@Before("BouncerFilter")
+public class AccessControlRequestFilter implements SecurityRequestFilter {
+ private final Set<String> allowedUrls;
+
+ @Inject
+ public AccessControlRequestFilter(HttpAccessControlConfig config) {
+ allowedUrls = Collections.unmodifiableSet(config.allowedUrls().stream().collect(Collectors.toSet()));
+ }
+
+ @Override
+ public void filter(DiscFilterRequest discFilterRequest, ResponseHandler responseHandler) {
+ String origin = discFilterRequest.getHeader("Origin");
+
+ if (!discFilterRequest.getMethod().equals(OPTIONS.name()))
+ return;
+
+ HttpResponse response = HttpResponse.newInstance(Response.Status.OK);
+
+ if (allowedUrls.contains(origin))
+ response.headers().add(ALLOW_ORIGIN_HEADER, origin);
+
+ ACCESS_CONTROL_HEADERS.forEach(
+ (name, value) -> response.headers().add(name, value));
+
+ ContentChannel cc = responseHandler.handleResponse(response);
+ cc.close(null);
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java
new file mode 100644
index 00000000000..c2ad31cd925
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilter.java
@@ -0,0 +1,55 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.yahoo.jdisc.AbstractResource;
+import com.yahoo.jdisc.http.filter.DiscFilterResponse;
+import com.yahoo.jdisc.http.filter.RequestView;
+import com.yahoo.jdisc.http.filter.SecurityResponseFilter;
+import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig;
+
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS;
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER;
+
+/**
+ * @author gv
+ * @author Tony Vaagenes
+ */
+public class AccessControlResponseFilter extends AbstractResource implements SecurityResponseFilter {
+
+ private final List<String> allowedUrls;
+
+ public AccessControlResponseFilter(HttpAccessControlConfig config) {
+ allowedUrls = config.allowedUrls();
+ }
+
+ @Override
+ public void filter(DiscFilterResponse response, RequestView request) {
+ Optional<String> requestOrigin = request.getFirstHeader("Origin");
+
+ requestOrigin.ifPresent(
+ origin -> allowedUrls.stream()
+ .filter(allowedUrl -> matchesRequestOrigin(origin, allowedUrl))
+ .findAny()
+ .ifPresent(allowedOrigin -> setHeaderUnlessExists(response, ALLOW_ORIGIN_HEADER, allowedOrigin))
+ );
+ ACCESS_CONTROL_HEADERS.forEach((name, value) -> setHeaderUnlessExists(response, name, value));
+ }
+
+ private boolean matchesRequestOrigin(String requestOrigin, String allowedUrl) {
+ return allowedUrl.equals("*") || requestOrigin.startsWith(allowedUrl);
+ }
+
+ /**
+ * This is to avoid duplicating headers already set by the {@link AccessControlRequestFilter}.
+ * Currently (March 2016), this filter is invoked for OPTIONS requests to jdisc request handlers,
+ * even if the request filter has been invoked first. For jersey based APIs, this filter is NOT
+ * invoked in these cases.
+ */
+ private void setHeaderUnlessExists(DiscFilterResponse response, String name, String value) {
+ if (response.getHeader(name) == null)
+ response.setHeader(name, value);
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java
new file mode 100644
index 00000000000..7beb3f755ad
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/DummyFilter.java
@@ -0,0 +1,16 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+
+/**
+ * @author Stian Kristoffersen
+ */
+public class DummyFilter implements SecurityRequestFilter {
+ @Override
+ public void filter(DiscFilterRequest request, ResponseHandler handler) {
+ /* Do nothing - a bug in JDisc prevents empty request chains */
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java
new file mode 100644
index 00000000000..0138d3ae65c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.google.inject.Inject;
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.yolean.chain.After;
+
+/**
+ * @author bjorncs
+ */
+@After("BouncerFilter")
+public class NTokenRequestFilter implements SecurityRequestFilter {
+
+ public static final String NTOKEN_HEADER = "com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter.ntoken";
+
+ private final Athens athens;
+
+ @Inject
+ public NTokenRequestFilter(Athens athens) {
+ this.athens = athens;
+ }
+
+ @Override
+ public void filter(DiscFilterRequest request, ResponseHandler responseHandler) {
+ String nToken = request.getHeader(athens.principalTokenHeader());
+ if (nToken != null) {
+ request.setAttribute(NTOKEN_HEADER, nToken);
+ }
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java
new file mode 100644
index 00000000000..7ea98528a88
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SetBouncerPassthruHeaderFilter.java
@@ -0,0 +1,27 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.yolean.chain.After;
+
+/**
+ * @author Stian Kristoffersen
+ */
+@After("BouncerFilter")
+public class SetBouncerPassthruHeaderFilter implements SecurityRequestFilter {
+
+ public static final String BOUNCER_PASSTHRU_ATTRIBUTE = "bouncer.bypassthru";
+ public static final String BOUNCER_PASSTHRU_COOKIE_OK = "1";
+ public static final String BOUNCER_PASSTHRU_HEADER_FIELD = "com.yahoo.hosted.vespa.bouncer.passthru";
+
+ @Override
+ public void filter(DiscFilterRequest request, ResponseHandler handler) {
+ Object statusProperty = request.getAttribute(BOUNCER_PASSTHRU_ATTRIBUTE);
+ String status = Integer.toString((int)statusProperty);
+
+ request.addHeader(BOUNCER_PASSTHRU_HEADER_FIELD, status);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java
new file mode 100644
index 00000000000..a88e881ce9d
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UnauthenticatedUserPrincipal.java
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import java.security.Principal;
+import java.util.Objects;
+
+/**
+ * A principal for an unauthenticated user (typically from a trusted host).
+ * This principal should only be used in combination with machine authentication!
+ *
+ * @author bjorncs
+ */
+public class UnauthenticatedUserPrincipal implements Principal {
+ private final String username;
+
+ public UnauthenticatedUserPrincipal(String username) {
+ this.username = username;
+ }
+
+ @Override
+ public String getName() {
+ return username;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UnauthenticatedUserPrincipal that = (UnauthenticatedUserPrincipal) o;
+ return Objects.equals(username, that.username);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(username);
+ }
+
+ @Override
+ public String toString() {
+ return "UnauthenticatedUserPrincipal{" +
+ "username='" + username + '\'' +
+ '}';
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java
new file mode 100644
index 00000000000..46df4d7a603
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/UserIdRequestFilter.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.vespa.hosted.controller.api.nonpublic.HeaderFields;
+import com.yahoo.yolean.chain.Before;
+
+/**
+ * Allows hosts using host-based authentication to set user ID.
+ *
+ * @author Tony Vaagenes
+ */
+@Before("CreateSecurityContextFilter")
+public class UserIdRequestFilter implements SecurityRequestFilter {
+
+ @Override
+ public void filter(DiscFilterRequest request, ResponseHandler handler) {
+ String userName = request.getHeader(HeaderFields.USER_ID_HEADER_FIELD);
+ request.setUserPrincipal(new UnauthenticatedUserPrincipal(userName));
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java
new file mode 100644
index 00000000000..850130ca970
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/CreateSecurityContextFilter.java
@@ -0,0 +1,50 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext;
+
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.vespa.hosted.controller.common.ContextAttributes;
+import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Provides;
+
+import javax.ws.rs.core.SecurityContext;
+import java.security.Principal;
+
+/**
+ * Exposes the security information from the disc filter request
+ * by storing a security context in the request context.
+ *
+ * @author Tony Vaagenes
+ */
+@After("BouncerFilter")
+@Provides("SecurityContext")
+public class CreateSecurityContextFilter implements SecurityRequestFilter {
+
+ @Override
+ public void filter(DiscFilterRequest request, ResponseHandler handler) {
+ request.setAttribute(ContextAttributes.SECURITY_CONTEXT_ATTRIBUTE,
+ new SecurityContext() {
+ @Override
+ public Principal getUserPrincipal() {
+ return request.getUserPrincipal();
+ }
+
+ @Override
+ public boolean isUserInRole(String role) {
+ return request.isUserInRole(role);
+ }
+
+ @Override
+ public boolean isSecure() {
+ return request.isSecure();
+ }
+
+ @Override
+ public String getAuthenticationScheme() {
+ throw new UnsupportedOperationException();
+ }
+ });
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java
new file mode 100644
index 00000000000..17c86e89362
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/PropagateSecurityContextFilter.java
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext;
+
+import com.yahoo.vespa.hosted.controller.common.ContextAttributes;
+
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+
+/**
+ * Get the security context from the underlying Servlet request, and expose it to
+ * Jersey resources.
+ *
+ * @author Tony Vaagenes
+ */
+@PreMatching
+@Provider
+public class PropagateSecurityContextFilter implements ContainerRequestFilter {
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+ SecurityContext securityContext =
+ (SecurityContext) requestContext.getProperty(ContextAttributes.SECURITY_CONTEXT_ATTRIBUTE);
+
+ if (securityContext != null) {
+ requestContext.setSecurityContext(securityContext);
+ }
+ }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java
new file mode 100644
index 00000000000..0b98599dbb0
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/securitycontext/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Jersey requires that the package is exported to be able to instantiate the filter.
+ *
+ * @author Tony Vaagenes
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java
new file mode 100644
index 00000000000..a623e880c4c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java
@@ -0,0 +1,168 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.screwdriver;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.io.IOUtils;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
+import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
+import com.yahoo.vespa.hosted.controller.restapi.StringResponse;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This implements a callback API from Screwdriver which lets deployment jobs notify the controller
+ * on completion.
+ *
+ * @author bratseth
+ */
+public class ScrewdriverApiHandler extends LoggingRequestHandler {
+
+ private final static Logger log = Logger.getLogger(ScrewdriverApiHandler.class.getName());
+
+ private final Controller controller;
+ // TODO: Remember to distinguish between PR jobs and component ones, by adding reports to the right jobs?
+
+ public ScrewdriverApiHandler(Executor executor, AccessLog accessLog, Controller controller) {
+ super(executor, accessLog);
+ this.controller = controller;
+ }
+
+ @Override
+ public HttpResponse handle(HttpRequest request) {
+ try {
+ Method method = request.getMethod();
+ String path = request.getUri().getPath();
+ switch (method) {
+ case GET: switch (path) {
+ case "/screwdriver/v1/release/vespa": return vespaVersion();
+ case "/screwdriver/v1/jobsToRun": return buildJobResponse(controller.applications().deploymentTrigger().buildSystem().jobs());
+ default: return ErrorResponse.notFoundError(String.format( "No '%s' handler at '%s'", method, path));
+ }
+ case POST: switch (path) {
+ case "/screwdriver/v1/jobreport": return handleJobReportPost(request);
+ default: return ErrorResponse.notFoundError(String.format( "No '%s' handler at '%s'", method, path));
+ }
+ case DELETE: switch (path) {
+ case "/screwdriver/v1/jobsToRun": return buildJobResponse(controller.applications().deploymentTrigger().buildSystem().takeJobsToRun());
+ default: return ErrorResponse.notFoundError(String.format( "No '%s' handler at '%s'", method, path));
+ }
+ default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
+ }
+ } catch (IllegalArgumentException|IllegalStateException e) {
+ return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+ } catch (RuntimeException e) {
+ log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+ return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+ }
+ }
+
+ private HttpResponse vespaVersion() {
+ VespaVersion version = controller.versionStatus().version(controller.systemVersion());
+ if (version == null)
+ return ErrorResponse.notFoundError("Information about the current system version is not available at this time");
+
+ Slime slime = new Slime();
+ Cursor cursor = slime.setObject();
+ cursor.setString("version", version.versionNumber().toString());
+ cursor.setString("sha", version.releaseCommit());
+ cursor.setLong("date", version.releasedAt().toEpochMilli());
+ return new SlimeJsonResponse(slime);
+
+ }
+
+ private HttpResponse buildJobResponse(List<BuildJob> buildJobs) {
+ Slime slime = new Slime();
+ Cursor buildJobArray = slime.setArray();
+ for (BuildJob buildJob : buildJobs) {
+ Cursor buildJobObject = buildJobArray.addObject();
+ buildJobObject.setLong("projectId", buildJob.projectId());
+ buildJobObject.setString("jobName", buildJob.jobName());
+ }
+ return new SlimeJsonResponse(slime);
+ }
+
+ /**
+ * Parse a JSON blob of the form:
+ * {
+ * "tenant" : String
+ * "application" : String
+ * "instance" : String
+ * "jobName" : String
+ * "projectId" : long
+ * "buildNumber" : long
+ * "success" : boolean
+ * "selfTriggering": boolean
+ * "gitChanges" : boolean
+ * "vespaVersion" : String
+ * }
+ * and notify the controller of the report.
+ *
+ * @param request The JSON blob.
+ * @return 200
+ */
+ private HttpResponse handleJobReportPost(HttpRequest request) {
+ // TODO: buildNumber is unused now -- remove, or use.
+ // TODO: selfTriggering is unused now -- remove, or use.
+ // TODO: gitChanges is unused now -- remove, or use.
+ // Note: gitChanges is probably only useful for the component step, since it check the gir repo directly;
+ // for other jobs, the last component's git commit is what matters.
+ // TODO: ApplicationId (tenant, application, instance) is unused now -- remove, or use.
+
+ controller.applications().notifyJobCompletion(toJobReport(toSlime(request.getData()).get()));
+
+ return new StringResponse("ok");
+ }
+
+ private Slime toSlime(InputStream jsonStream) {
+ try {
+ byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000);
+ return SlimeUtils.jsonToSlime(jsonBytes);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private JobReport toJobReport(Inspector report) {
+ Optional<JobError> jobError = Optional.empty();
+ if (report.field("jobError").valid()) {
+ jobError = Optional.of(JobError.valueOf(report.field("jobError").asString()));
+ } else if (report.field("success").valid()) { // TODO: Remove after May 2017
+ jobError = JobError.from(report.field("success").asBool());
+ }
+ return new JobReport(
+ ApplicationId.from(
+ report.field("tenant").asString(),
+ report.field("application").asString(),
+ report.field("instance").asString()),
+ JobType.fromId(report.field("jobName").asString()),
+ report.field("projectId").asLong(),
+ report.field("buildNumber").asLong(),
+ jobError,
+ report.field("selfTriggering").asBool(),
+ report.field("gitChanges").asBool()
+ );
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
new file mode 100644
index 00000000000..fbd1a74c12c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
@@ -0,0 +1,62 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+
+import java.util.List;
+
+/**
+ * Statistics about deployments on a platform version. This is immutable.
+ *
+ * @author bratseth
+ */
+public class DeploymentStatistics {
+
+ private final Version version;
+ private final ImmutableList<ApplicationId> failing;
+ private final ImmutableList<ApplicationId> production;
+
+ private DeploymentStatistics(Version version,
+ List<ApplicationId> failingApplications, List<ApplicationId> production) {
+ this.version = version;
+ this.failing = ImmutableList.copyOf(failingApplications);
+ this.production = ImmutableList.copyOf(production);
+ }
+
+ /** Returns a statistics instance with the values as 0 */
+ public static DeploymentStatistics empty(Version version) {
+ return new DeploymentStatistics(version, ImmutableList.of(), ImmutableList.of());
+ }
+
+ /** Returns the version these statistics are for */
+ public Version version() { return version; }
+
+ /**
+ * Returns the applications which have at least one job (of any type) which fails on this version,
+ * excluding errors known to not be caused by this version
+ */
+ public List<ApplicationId> failing() { return failing; }
+
+ /** Returns the applications which have this version in production in at least one zone */
+ public List<ApplicationId> production() { return production; }
+
+ /** Returns a version of this with the given failing application added */
+ public DeploymentStatistics withFailing(ApplicationId application) {
+ return new DeploymentStatistics(version, add(application, failing), production);
+ }
+
+ /** Returns a version of this with the given production application added */
+ public DeploymentStatistics withProduction(ApplicationId application) {
+ return new DeploymentStatistics(version, failing, add(application, production));
+ }
+
+ private ImmutableList<ApplicationId> add(ApplicationId application, ImmutableList<ApplicationId> list) {
+ ImmutableList.Builder<ApplicationId> b = new ImmutableList.Builder<>();
+ b.addAll(list);
+ b.add(application);
+ return b.build();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
new file mode 100644
index 00000000000..bef96014e79
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -0,0 +1,182 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.collections.ListMap;
+import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
+import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+
+import java.net.URI;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * Information about the current platform versions in use.
+ * The versions in use are the set of all versions running in current applications, versions
+ * of config servers in all zones, and the version of this controller itself.
+ *
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class VersionStatus {
+
+ private static final Logger log = Logger.getLogger(VersionStatus.class.getName());
+
+ private static final String VESPA_REPO = "vespa-yahoo";
+ private static final String VESPA_REPO_OWNER = "vespa";
+
+ private final ImmutableList<VespaVersion> versions;
+
+ /** Create a version status. DO NOT USE: Public for testing only */
+ public VersionStatus(List<VespaVersion> versions) {
+ this.versions = ImmutableList.copyOf(versions);
+ }
+
+ /**
+ * Returns the current Vespa version of the system controlled by this,
+ * or empty if we have not currently determined what the system version is in this status.
+ */
+ public Optional<VespaVersion> systemVersion() {
+ return versions().stream().filter(VespaVersion::isCurrentSystemVersion).findAny();
+ }
+
+ /**
+ * Lists all currently active Vespa versions, with deployment statistics,
+ * sorted from lowest to highest version number.
+ * The returned list is immutable.
+ * Calling this is free, but the returned status is slightly out of date.
+ */
+ public List<VespaVersion> versions() { return versions; }
+
+ /** Returns the given version, or null if it is not present */
+ public VespaVersion version(Version version) {
+ return versions.stream().filter(v -> v.versionNumber().equals(version)).findFirst().orElse(null);
+ }
+
+ /** Create the empty version status */
+ public static VersionStatus empty() { return new VersionStatus(ImmutableList.of()); }
+
+ /** Create a full, updated version status. This is expensive and should be done infrequently */
+ public static VersionStatus compute(Controller controller) {
+ return compute(controller, Vtag.currentVersion);
+ }
+
+ /** Compute version status using the given current version. This is useful for testing. */
+ public static VersionStatus compute(Controller controller, Version currentVersion) {
+ ListMap<Version, String> configServerVersions = findConfigServerVersions(controller);
+
+ Set<Version> infrastructureVersions = new HashSet<>();
+ infrastructureVersions.add(currentVersion);
+ infrastructureVersions.addAll(configServerVersions.keySet());
+
+ // The system version is the oldest infrastructure version
+ Version systemVersion = infrastructureVersions.stream().sorted().findFirst().get();
+
+ Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions,
+ controller.applications().asList());
+ List<VespaVersion> versions = new ArrayList<>();
+
+ for (DeploymentStatistics statistics : deploymentStatistics) {
+ if (statistics.version().isEmpty()) continue;
+
+ try {
+ VespaVersion vespaVersion = createVersion(statistics,
+ statistics.version().equals(systemVersion),
+ configServerVersions.getList(statistics.version()),
+ controller);
+ versions.add(vespaVersion);
+ } catch (IllegalArgumentException e) {
+ log.log(Level.WARNING, "Unable to create VespaVersion for version " +
+ statistics.version().toFullString(), e);
+ }
+ }
+ Collections.sort(versions);
+
+ return new VersionStatus(versions);
+ }
+
+ private static ListMap<Version, String> findConfigServerVersions(Controller controller) {
+ List<URI> configServers = controller.zoneRegistry().zones().stream()
+ .flatMap(zone -> controller.getConfigServerUris(zone.environment(), zone.region()).stream())
+ .collect(Collectors.toList());
+
+ ListMap<Version, String> versions = new ListMap<>();
+ for (URI configServer : configServers)
+ versions.put(controller.applications().configserverClient().version(configServer), configServer.getHost());
+ return versions;
+ }
+
+ private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions,
+ List<Application> applications) {
+ Map<Version, DeploymentStatistics> versionMap = new HashMap<>();
+
+ for (Version infrastructureVersion : infrastructureVersions)
+ versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion));
+
+ for (Application application : applications) {
+ DeploymentJobs jobs = application.deploymentJobs();
+
+ // Note that each version deployed on this application exists
+ for (Deployment deployment : application.deployments().values())
+ versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty);
+
+ // List versions which have failing jobs, and versions which are in production
+ // TODO: Don't count applications which started failing on an application change, not a version change
+
+ // Failing versions
+ Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream()
+ .filter(jobStatus -> jobStatus.lastCompleted().isPresent())
+ .filter(jobStatus -> jobStatus.jobError().isPresent())
+ .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity)
+ .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version()));
+ for (Version v : failingJobsByVersion.keySet()) {
+ versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withFailing(application.id()));
+ }
+
+ // Succeeding versions
+ Map<Version, List<JobStatus>> succeedingJobsByVersions = jobs.jobStatus().values().stream()
+ .filter(jobStatus -> jobStatus.lastSuccess().isPresent())
+ .filter(jobStatus -> jobStatus.type().isProduction())
+ .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastSuccess().get().version()));
+ for (Version v : succeedingJobsByVersions.keySet()) {
+ versionMap.compute(v, (version, statistics) -> emptyIfMissing(version, statistics).withProduction(application.id()));
+ }
+ }
+ return versionMap.values();
+ }
+
+ private static DeploymentStatistics emptyIfMissing(Version version, DeploymentStatistics statistics) {
+ return statistics == null ? DeploymentStatistics.empty(version) : statistics;
+ }
+
+ private static VespaVersion createVersion(DeploymentStatistics statistics,
+ boolean isSystemVersion,
+ Collection<String> configServerHostnames,
+ Controller controller) {
+ GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString());
+ return new VespaVersion(statistics,
+ gitSha.sha, Instant.ofEpochMilli(gitSha.commit.author.date.getTime()),
+ isSystemVersion,
+ configServerHostnames,
+ controller);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
new file mode 100644
index 00000000000..ce5533bd0bc
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -0,0 +1,139 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
+import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * Information about a particular Vespa version.
+ * VespaVersions are identified by their version number and ordered by increasing version numbers.
+ *
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+public class VespaVersion implements Comparable<VespaVersion> {
+
+ private final String releaseCommit;
+ private final Instant releasedAt;
+ private final boolean isCurrentSystemVersion;
+ private final DeploymentStatistics statistics;
+ private final Confidence confidence;
+ private final ImmutableSet<String> configServerHostnames;
+
+ public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt,
+ boolean isCurrentSystemVersion, Collection<String> configServerHostnames,
+ Controller controller) {
+ this.statistics = statistics;
+ this.releaseCommit = releaseCommit;
+ this.releasedAt = releasedAt;
+ this.isCurrentSystemVersion = isCurrentSystemVersion;
+ this.configServerHostnames = ImmutableSet.copyOf(configServerHostnames);
+ this.confidence = deduceConfidenceFrom(statistics, controller, releasedAt);
+ }
+
+ private static Confidence deduceConfidenceFrom(DeploymentStatistics statistics,
+ Controller controller,
+ Instant releasedAt) {
+ // 'production on this': All deployment jobs upgrading to this version have completed without failure
+ ApplicationList productionOnThis = ApplicationList.from(statistics.production(), controller.applications())
+ .notUpgradingTo(statistics.version())
+ .notFailing();
+ ApplicationList failingOnThis = ApplicationList.from(statistics.failing(), controller.applications());
+ ApplicationList all = ApplicationList.from(controller.applications().asList())
+ .hasDeployment()
+ .notPullRequest();
+
+ // 'broken' if any Canary fails
+ if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty())
+ return Confidence.broken;
+
+ // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all
+ int brokenByThisVersion = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt).size();
+ if (brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1)
+ return Confidence.broken;
+
+ // 'low' unless all canary applications are upgraded
+ if (productionOnThis.with(UpgradePolicy.canary).size() < all.with(UpgradePolicy.canary).size())
+ return Confidence.low;
+
+ // 'high' if 90% of all default upgrade applications upgraded
+ if (productionOnThis.with(UpgradePolicy.defaultPolicy).size() >=
+ all.with(UpgradePolicy.defaultPolicy).size() * 0.9)
+ return Confidence.high;
+
+ return Confidence.normal;
+ }
+
+ /** Returns the version number of this Vespa version */
+ public Version versionNumber() { return statistics.version(); }
+
+ /** Returns the sha of the release tag commit for this version in git */
+ public String releaseCommit() { return releaseCommit; }
+
+ /** Returns the time of the release commit */
+ public Instant releasedAt() { return releasedAt; }
+
+ /** Statistics about deployment of this version */
+ public DeploymentStatistics statistics() { return statistics; }
+
+ /** Returns whether this is the version currently running on this controller */
+ public boolean isSelfVersion() { return versionNumber().equals(Vtag.currentVersion); }
+
+ /**
+ * Returns whether this is the current version of the infrastructure of the system
+ * (i.e the lowest version across this controller and all config servers in all zones).
+ * A goal of the controller is to eventually (limited by safety and upgrade capacity) drive
+ * all applications to this version.
+ *
+ * Note that the self version may be higher than the current system version if
+ * all config servers are not yet upgraded to the version of this controller.
+ */
+ public boolean isCurrentSystemVersion() { return isCurrentSystemVersion; }
+
+ /** Returns the host names of the config servers (across all zones) which are currently of this version */
+ public Set<String> configServerHostnames() { return configServerHostnames; }
+
+ /** Returns the confidence we have in this versions suitability for production */
+ public Confidence confidence() { return confidence; }
+
+ @Override
+ public int compareTo(VespaVersion other) {
+ return this.versionNumber().compareTo(other.versionNumber());
+ }
+
+ @Override
+ public int hashCode() { return versionNumber().hashCode(); }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) return true;
+ if ( ! (other instanceof VespaVersion)) return false;
+ return ((VespaVersion)other).versionNumber().equals(this.versionNumber());
+ }
+
+ public enum Confidence {
+
+ /** This version has been proven defective */
+ broken,
+
+ /** We don't have sufficient evidence that this version is working */
+ low,
+
+ /** We have sufficient evidence that this version is working */
+ normal,
+
+ /** We have overwhelming evidence that this version is working */
+ high
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java
new file mode 100644
index 00000000000..f5852b9dfcf
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResource.java
@@ -0,0 +1,55 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.restapi.impl;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.inject.Inject;
+import com.yahoo.container.jaxrs.annotation.Component;
+import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService;
+
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriBuilder;
+
+/**
+ * Proxies requests from controller to https://xxx.statuspage.io/api/v2/yyy.json?api_key=zzz[&amp;since=YYYY-MM-DDThh:mm[:ss]±hh:mm]
+ *
+ * @author andreer
+ */
+@Path("/v1/")
+@Produces(MediaType.APPLICATION_JSON)
+public class StatusPageResource implements com.yahoo.vespa.hosted.controller.api.statuspage.StatusPageResource {
+
+ private final Client client;
+ private final KeyService keyService;
+
+ @Inject
+ public StatusPageResource(@Component KeyService keyService) {
+ this(keyService, ClientBuilder.newClient());
+ }
+
+ protected StatusPageResource(KeyService keyService, Client client) {
+ this.keyService = keyService;
+ this.client = client;
+ }
+
+ protected UriBuilder statusPageURL(String page, String since) {
+ String[] secrets = keyService.getSecret("vespa_hosted.controller.statuspage_api_key").split(":");
+ UriBuilder uriBuilder = UriBuilder.fromUri("https://" + secrets[0] + ".statuspage.io/api/v2/" + page + ".json?api_key=" + secrets[1]);
+ if (since != null) {
+ uriBuilder.queryParam("since", since);
+ }
+
+ return uriBuilder;
+ }
+
+ @Override
+ public JsonNode statusPage(String page, String since) {
+ WebTarget target = client.target(statusPageURL(page, since));
+ return target.request().get(JsonNode.class);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java
new file mode 100644
index 00000000000..dca8a22a313
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/restapi/impl/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author Tony Vaagenes
+ */
+@ExportPackage
+package com.yahoo.vespa.hosted.restapi.impl;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java
new file mode 100644
index 00000000000..9eef1dac70b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java
@@ -0,0 +1,148 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.rotation;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.log.LogLevel;
+import com.yahoo.metrics.simple.Gauge;
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId;
+import com.yahoo.vespa.hosted.controller.api.ApplicationAlias;
+import com.yahoo.vespa.hosted.controller.persistence.ControllerDb;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * A rotation repository.
+ *
+ * @author Oyvind Gronnesby
+ */
+// TODO: Fold this into ApplicationController+Application
+public class ControllerRotationRepository implements RotationRepository {
+
+ private static final Logger log = Logger.getLogger(ControllerRotationRepository.class.getName());
+
+ private static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations";
+ private final Gauge remainingRotations;
+
+ private final ControllerDb controllerDb;
+ private final Map<RotationId, Rotation> rotationsMap;
+
+ public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, MetricReceiver metricReceiver) {
+ this.controllerDb = controllerDb;
+ this.rotationsMap = buildRotationsMap(rotationConfig);
+ this.remainingRotations = metricReceiver.declareGauge(REMAINING_ROTATIONS_METRIC_NAME);
+ }
+
+ private static Map<RotationId, Rotation> buildRotationsMap(RotationsConfig rotationConfig) {
+ return rotationConfig.rotations().entrySet().stream()
+ .map(entry -> {
+ RotationId rotationId = new RotationId(entry.getKey());
+ return new Rotation(rotationId, entry.getValue().trim());
+ })
+ .collect(Collectors.toMap(
+ rotation -> rotation.rotationId,
+ rotation -> rotation
+ ));
+ }
+
+ @Override
+ @NotNull
+ public Set<Rotation> getOrAssignRotation(ApplicationId applicationId, DeploymentSpec deploymentSpec) {
+ reportRemainingRotations();
+
+ Set<RotationId> rotations = controllerDb.getRotations(applicationId);
+
+ if (rotations.size() > 1) {
+ log.warning(String.format("Application %s has %d > 1 rotation", applicationId, rotations.size()));
+ }
+
+ if (!rotations.isEmpty()) {
+ return rotations.stream()
+ .map(rotationsMap::get)
+ .collect(Collectors.toSet());
+ }
+
+ if( ! deploymentSpec.globalServiceId().isPresent()) {
+ return Collections.emptySet();
+ }
+
+ long productionZoneCount = deploymentSpec.zones().stream()
+ .filter(zone -> zone.deploysTo(Environment.prod))
+ .filter(zone -> ! isCorp(zone)) // Global rotations don't work for nodes in corp network
+ .count();
+
+ if (productionZoneCount >= 2) {
+ return assignRotation(applicationId);
+ }
+ else {
+ throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined");
+ }
+ }
+
+ private boolean isCorp(DeploymentSpec.DeclaredZone zone) {
+ return zone.region().isPresent() && zone.region().get().value().contains("corp");
+ }
+
+ @Override
+ @NotNull
+ public Set<URI> getRotationUris(ApplicationId applicationId) {
+ Set<RotationId> rotations = controllerDb.getRotations(applicationId);
+ if (rotations.isEmpty()) {
+ return Collections.emptySet();
+ }
+ else {
+ ApplicationAlias applicationAlias = new ApplicationAlias(applicationId);
+ return Collections.singleton(applicationAlias.toHttpUri());
+ }
+ }
+
+ private Set<Rotation> assignRotation(ApplicationId applicationId) {
+ Set<RotationId> availableRotations = availableRotations();
+ if (availableRotations.isEmpty()) {
+ String message = "Unable to assign global rotation to "
+ + applicationId + " - no rotations available";
+ log.info(message);
+ throw new RuntimeException(message);
+ }
+
+ for (RotationId rotationId : availableRotations) {
+ if (controllerDb.assignRotation(rotationId, applicationId)) {
+ log.info(String.format("Assigned rotation %s to application %s", rotationId, applicationId));
+ Rotation rotation = this.rotationsMap.get(rotationId);
+ return Collections.singleton(rotation);
+ }
+ }
+
+ log.info(String.format("Rotation: No rotations assigned with %s rotations available", availableRotations.size()));
+ return Collections.emptySet();
+ }
+
+ private Set<RotationId> availableRotations() {
+ Set<RotationId> assignedRotations = controllerDb.getRotations();
+ Set<RotationId> allRotations = new HashSet<>(rotationsMap.keySet());
+ allRotations.removeAll(assignedRotations);
+ return allRotations;
+ }
+
+ private void reportRemainingRotations() {
+ try {
+ int freeRotationsCount = availableRotations().size();
+ log.log(LogLevel.INFO, "Rotation: {0} global rotations remaining", freeRotationsCount);
+ remainingRotations.sample(freeRotationsCount);
+ } catch (Exception e) {
+ log.log(LogLevel.INFO, "Failed to report rotations metric", e);
+ }
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java
new file mode 100644
index 00000000000..4e333f0268b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java
@@ -0,0 +1,54 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.rotation;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * A rotation repository backed by in-memory data structures
+ *
+ * @author bratseth
+ */
+public class MemoryRotationRepository implements RotationRepository {
+
+ private final Map<ApplicationId, Set<Rotation>> rotations = new HashMap<>();
+
+ @NotNull
+ @Override
+ public Set<Rotation> getOrAssignRotation(ApplicationId application, DeploymentSpec deploymentSpec) {
+ if (rotations.containsKey(application)) {
+ return rotations.get(application);
+ }
+ Set<Rotation> rotations = ImmutableSet.of(new Rotation(
+ new RotationId("generated-by-routing-service-" + UUID.randomUUID().toString()),
+ "fake-global-rotation-" + application.toShortString())
+ );
+ this.rotations.put(application, rotations);
+ return rotations;
+ }
+
+ @NotNull
+ @Override
+ public Set<URI> getRotationUris(ApplicationId applicationId) {
+ Set<Rotation> rotations = this.rotations.get(applicationId);
+ if (rotations == null) {
+ return Collections.emptySet();
+ }
+ return rotations.stream()
+ .map(rotation -> URI.create("http://" + rotation.rotationName))
+ .collect(Collectors.toSet());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java
new file mode 100644
index 00000000000..b1f7b33e58e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java
@@ -0,0 +1,48 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.rotation;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.util.Set;
+
+/**
+ * A rotation repository assigns global rotations to Vespa applications. It does not take into account
+ * whether an application qualifies or not, but it assumes that each application should get only
+ * one.
+ *
+ * The list of rotations comes from the RotationsConfig, set in the controller's services.xml.
+ * Assignments are persisted with the RotationId as the primary key. When we assign the
+ * rotation to an application we try to put the mapping RotationId -&gt; Application. If a
+ * mapping already exists for that RotationId, the assignment will fail.
+ *
+ * @author Oyvind Gronnesby
+ */
+public interface RotationRepository {
+
+ // TODO: Change to use provision.ApplicationId
+ // TODO: Move the persistence into ControllerDb (done), and then collapse the 2 implementations and the interface into one
+
+ /**
+ * If any rotations are assigned to the application, these will be returned.
+ * If no rotations are assigned, assign one rotation to the application and return that.
+ *
+ * @param applicationId ID of the application to get or assign rotation for
+ * @param deploymentSpec Spec of current application being deployed
+ * @return Set of rotations assigned (may be empty)
+ */
+ @NotNull
+ Set<Rotation> getOrAssignRotation(ApplicationId applicationId, DeploymentSpec deploymentSpec);
+
+ /**
+ * Get the external visible rotation URIs for this application.
+ *
+ * @param applicationId ID of the application to get or assign rotation for
+ */
+ @NotNull
+ Set<URI> getRotationUris(ApplicationId applicationId);
+
+}
diff --git a/controller-server/src/main/resources/WEB-INF/web.xml b/controller-server/src/main/resources/WEB-INF/web.xml
new file mode 100644
index 00000000000..f294a8eb46e
--- /dev/null
+++ b/controller-server/src/main/resources/WEB-INF/web.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<jdisc-config>
+ <init-param>
+ <param-name>com.sun.jersey.spi.container.ContainerRequestFilters</param-name>
+ <param-value>com.yahoo.vespa.hosted.controller.restapi.filter.securitycontext.PropagateSecurityContextFilter</param-value>
+ </init-param>
+</jdisc-config>
diff --git a/controller-server/src/main/resources/configdefinitions/http-access-control.def b/controller-server/src/main/resources/configdefinitions/http-access-control.def
new file mode 100644
index 00000000000..4cd1532761b
--- /dev/null
+++ b/controller-server/src/main/resources/configdefinitions/http-access-control.def
@@ -0,0 +1,4 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.hosted.controller.restapi.filter.config
+
+allowedUrls[] string
diff --git a/controller-server/src/main/resources/configdefinitions/maintainer.def b/controller-server/src/main/resources/configdefinitions/maintainer.def
new file mode 100644
index 00000000000..7ec8860bef4
--- /dev/null
+++ b/controller-server/src/main/resources/configdefinitions/maintainer.def
@@ -0,0 +1,4 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.hosted.controller.maintenance.config
+
+intervalMinutes int default=30
diff --git a/controller-server/src/main/resources/configdefinitions/rotations.def b/controller-server/src/main/resources/configdefinitions/rotations.def
new file mode 100644
index 00000000000..d4f3636d0d8
--- /dev/null
+++ b/controller-server/src/main/resources/configdefinitions/rotations.def
@@ -0,0 +1,4 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=vespa.hosted.rotation.config
+
+rotations{} string
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
new file mode 100644
index 00000000000..6018c99206e
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
@@ -0,0 +1,204 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+import com.yahoo.vespa.serviceview.bindings.ClusterView;
+import com.yahoo.vespa.serviceview.bindings.ServiceView;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * @author mortent
+ */
+public class ConfigServerClientMock extends AbstractComponent implements ConfigServerClient {
+
+ private Map<ApplicationId, byte[]> applicationContent = new HashMap<>();
+ private Map<ApplicationId, String> applicationInstances = new HashMap<>();
+ private Map<ApplicationId, Boolean> applicationActivated = new HashMap<>();
+ private Set<ApplicationId> applicationRestarted = new HashSet<>();
+ private Set<String> hostsExplicitlyRestarted = new HashSet<>();
+ private Map<String, EndpointStatus> endpoints = new HashMap<>();
+
+ private Map<URI, Version> configServerVersions = new HashMap<>();
+ private Version defaultConfigServerVersion = new Version(6, 1, 0);
+
+ /** The exception to throw on the next prepare run, or null to continue normally */
+ private RuntimeException prepareException = null;
+
+ /** The version given in the previous prepare call, or null if no call has been made */
+ public Optional<Version> lastPrepareVersion = null;
+
+ @Override
+ public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) {
+ lastPrepareVersion = deployOptions.vespaVersion.map(Version::new);
+
+ if (prepareException != null) {
+ RuntimeException prepareException = this.prepareException;
+ this.prepareException = null;
+ throw prepareException;
+ }
+
+ applicationContent.put(deployment.applicationId(), content);
+ applicationActivated.put(deployment.applicationId(), false);
+ applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080");
+
+ return new PreparedApplication() {
+ @Override
+ public void activate() {
+ applicationActivated.put(deployment.applicationId(), true);
+ }
+
+ @Override
+ public List<Log> messages() {
+ Log warning = new Log();
+ warning.level = "WARNING";
+ warning.time = 1;
+ warning.message = "The warning";
+
+ Log info = new Log();
+ info.level = "INFO";
+ info.time = 2;
+ info.message = "The info";
+
+ return Arrays.asList(warning, info);
+ }
+
+ @Override
+ public PrepareResponse prepareResponse() {
+ PrepareResponse prepareResponse = new PrepareResponse();
+ prepareResponse.message = "foo";
+ prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList());
+ prepareResponse.tenant = new TenantId("tenant");
+ return prepareResponse;
+ }
+ };
+ }
+
+ public void throwOnNextPrepare(RuntimeException prepareException) {
+ this.prepareException = prepareException;
+ }
+
+ /**
+ * Returns the (initially empty) mutable map of config server urls to versions.
+ * This API will return defaultConfigserverVersion as response to any version(url) call for versions not added to the map.
+ */
+ public Map<URI, Version> configServerVersions() {
+ return configServerVersions;
+ }
+
+ public Version getDefaultConfigServerVersion() { return defaultConfigServerVersion; }
+ public void setDefaultConfigServerVersion(Version version) { defaultConfigServerVersion = version; }
+
+ @Override
+ public List<String> getNodeQueryHost(DeploymentId deployment, String type) {
+ if (applicationInstances.containsKey(deployment.applicationId())) {
+ return Collections.singletonList(applicationInstances.get(deployment.applicationId()));
+ } else {
+ return Collections.emptyList();
+ }
+ }
+
+ @Override
+ public void restart(DeploymentId deployment, Optional<Hostname> hostname) {
+ applicationRestarted.add(deployment.applicationId());
+ if (hostname.isPresent()) {
+ hostsExplicitlyRestarted.add(hostname.get().id());
+ }
+ }
+
+ @Override
+ public void deactivate(DeploymentId deployment) {
+ applicationActivated.remove(deployment.applicationId());
+ applicationContent.remove(deployment.applicationId());
+ applicationInstances.remove(deployment.applicationId());
+ }
+
+ @Override
+ public JsonNode waitForConfigConverge(DeploymentId applicationInstance, long timeoutInSeconds) {
+ ObjectNode root = new ObjectNode(JsonNodeFactory.instance);
+ root.put("generation", 1);
+ return root;
+ }
+
+ @Override
+ public JsonNode grabLog(DeploymentId applicationInstance) {
+ return new ObjectNode(JsonNodeFactory.instance);
+ }
+
+ // Returns a canned example response
+ @Override
+ public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) {
+ ApplicationView applicationView = new ApplicationView();
+ ClusterView cluster = new ClusterView();
+ cluster.name = "cluster1";
+ cluster.type = "content";
+ cluster.url = "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1";
+ ServiceView service = new ServiceView();
+ service.configId = "cluster1/storage/0";
+ service.host = "host1";
+ service.serviceName = "storagenode";
+ service.serviceType = "storagenode";
+ service.url = "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/";
+ cluster.services = new ArrayList<>();
+ cluster.services.add(service);
+ applicationView.clusters = new ArrayList<>();
+ applicationView.clusters.add(cluster);
+ return applicationView;
+ }
+
+ // Returns a canned example response
+ @Override
+ public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath) {
+ Map<String,List<?>> root = new HashMap<>();
+ List<Map<?,?>> resources = new ArrayList<>();
+ Map<String,String> resource = new HashMap<>();
+ resource.put("url", "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config");
+ resources.add(resource);
+ root.put("resources", resources);
+ return root;
+ }
+
+ @Override
+ public Version version(URI configServerURI) {
+ return configServerVersions.getOrDefault(configServerURI, defaultConfigServerVersion);
+ }
+
+ @Override
+ public void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status) {
+ endpoints.put(endpoint, status);
+ }
+
+ @Override
+ public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) {
+ EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757l);
+ return endpoints.containsKey(endpoint)
+ ? endpoints.get(endpoint)
+ : result;
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
new file mode 100644
index 00000000000..e807762371a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -0,0 +1,601 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.BuildSystem;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
+import static org.fest.assertions.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author bratseth
+ * @author mpolden
+ */
+public class ControllerTest {
+
+ private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+
+ @Test
+ public void testDeployment() {
+ // Setup system
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationController applications = tester.controller().applications();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .region("us-east-3")
+ .build();
+
+ // staging job - succeeding
+ Version version1 = Version.fromString("6.1"); // Set in config server mock
+ Application app1 = tester.createApplication("app1", "tenant1", 1, 11L);
+ applications.notifyJobCompletion(mockReport(app1, component, true, false));
+ assertFalse("Revision is currently not known",
+ ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent());
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+ assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
+
+ Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision();
+ assertTrue("Revision has been set during deployment", revision.isPresent());
+ assertStatus(JobStatus.initial(stagingTest)
+ .withTriggering(version1, revision, tester.clock().instant())
+ .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller());
+
+ // Causes first deployment job to be triggered
+ assertStatus(JobStatus.initial(productionCorpUsEast1)
+ .withTriggering(version1, revision, tester.clock().instant()), app1.id(), tester.controller());
+ tester.clock().advance(Duration.ofSeconds(1));
+
+ // production job (failing)
+ tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, false);
+ assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
+
+ JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1)
+ .withTriggering(version1, revision, tester.clock().instant()) // Triggered first without revision info
+ .withCompletion(Optional.of(JobError.unknown), tester.clock().instant(), tester.controller())
+ .withTriggering(version1, revision, tester.clock().instant()); // Re-triggering (due to failure) has revision info
+
+ assertStatus(expectedJobStatus, app1.id(), tester.controller());
+
+ // Simulate restart
+ tester.restartController();
+ applications = tester.controller().applications();
+
+ assertNotNull(tester.controller().tenants().tenant(new TenantId("tenant1")));
+ assertNotNull(applications.get(ApplicationId.from(TenantName.from("tenant1"),
+ ApplicationName.from("application1"),
+ InstanceName.from("default"))));
+ assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
+
+ tester.clock().advance(Duration.ofSeconds(1));
+
+ // system and staging test job - succeeding
+ applications.notifyJobCompletion(mockReport(app1, component, true, false));
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ assertStatus(JobStatus.initial(systemTest)
+ .withTriggering(version1, revision, tester.clock().instant())
+ .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller());
+ tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+
+ // production job succeeding now
+ tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, true);
+ expectedJobStatus = expectedJobStatus
+ .withTriggering(version1, revision, tester.clock().instant())
+ .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller());
+ assertStatus(expectedJobStatus, app1.id(), tester.controller());
+
+ // causes triggering of next production job
+ assertStatus(JobStatus.initial(productionUsEast3)
+ .withTriggering( version1, revision, tester.clock().instant()),
+ app1.id(), tester.controller());
+ tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+
+ assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size());
+
+ // prod zone removal is not allowed
+ applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+ applications.notifyJobCompletion(mockReport(app1, component, true, false));
+ try {
+ tester.deploy(systemTest, app1, applicationPackage);
+ fail("Expected exception due to unallowed production deployment removal");
+ }
+ catch (IllegalArgumentException e) {
+ assertEquals("deployment-removal: application 'tenant1.app1' is deployed in corp-us-east-1, but does not include this zone in deployment.xml", e.getMessage());
+ }
+ assertNotNull("Zone was not removed",
+ applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get()));
+ assertNotNull("Deployment job was not removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1));
+
+ // prod zone removal is allowed with override
+ applicationPackage = new ApplicationPackageBuilder()
+ .allow(ValidationId.deploymentRemoval)
+ .upgradePolicy("default")
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ assertNull("Zone was removed",
+ applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get()));
+ assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1));
+ }
+
+ @Test
+ public void testDeployVersion() {
+ // Setup system
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationController applications = tester.controller().applications();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .build();
+ Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber();
+
+ Application app1 = tester.createApplication("application1", "tenant1", 1, 1L);
+ applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(false)), applications.lock(app1.id()));
+
+ // First deployment: An application change
+ applications.notifyJobCompletion(mockReport(app1, component, true, false));
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+ tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+
+ app1 = applications.require(app1.id());
+ assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get());
+ assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get());
+
+ // Unexpected deployment
+ try {
+ tester.deploy(productionUsWest1, app1, applicationPackage);
+ fail("Expected exception as no change was to be deployed");
+ }
+ catch (IllegalArgumentException expected) {
+ // success
+ }
+
+ // Application change after a new system version, and a region added
+ Version newSystemVersion = incrementSystemVersion(tester.controller());
+ assertTrue(newSystemVersion.isAfter(systemVersion));
+
+ applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .region("us-east-3")
+ .build();
+ applications.notifyJobCompletion(mockReport(app1, component, true, false));
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+ tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+
+ app1 = applications.require(app1.id());
+ assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
+ assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get());
+
+ // A deployment to the new region gets the same version
+ applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .region("us-east-3")
+ .build();
+ tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+ app1 = applications.require(app1.id());
+ assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
+ assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get());
+
+ // Version upgrade changes system version
+ Change.VersionChange change = new Change.VersionChange(newSystemVersion);
+ applications.deploymentTrigger().triggerChange(app1.id(), change);
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+ tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+ tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+
+ app1 = applications.require(app1.id());
+ assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get());
+ assertEquals(newSystemVersion, tester.configServerClientMock().lastPrepareVersion.get());
+ }
+
+ /** Adds a new version, higher than the current system version, makes it the system version and returns it */
+ private Version incrementSystemVersion(Controller controller) {
+ Version systemVersion = controller.versionStatus().systemVersion().get().versionNumber();
+ Version newSystemVersion = new Version(systemVersion.getMajor(), systemVersion.getMinor()+1, 0);
+ VespaVersion newSystemVespaVersion = new VespaVersion(DeploymentStatistics.empty(newSystemVersion),
+ "commit1",
+ Instant.now(),
+ true,
+ Collections.emptyList(),
+ controller);
+ List<VespaVersion> versions = new ArrayList<>(controller.versionStatus().versions());
+ for (int i = 0; i < versions.size(); i++) {
+ VespaVersion c = versions.get(i);
+ if (c.isCurrentSystemVersion())
+ versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), false, c.configServerHostnames(), controller));
+ }
+ versions.add(newSystemVespaVersion);
+ controller.updateVersionStatus(new VersionStatus(versions));
+ return newSystemVersion;
+ }
+
+ @Test
+ public void testPullRequestDeployment() {
+ // Setup system
+ ControllerTester tester = new ControllerTester();
+ ApplicationController applications = tester.controller().applications();
+
+ // staging deployment
+ long app1ProjectId = 22;
+ ApplicationId app1 = tester.createAndDeploy("tenant1", "domain1", "application1", Environment.staging, app1ProjectId).id();
+
+ // pull-request deployment - uses different instance id
+ ApplicationId app1pr = tester.createAndDeploy("tenant1", "domain1", "application1", "default-pr1", Environment.staging, app1ProjectId, null).id();
+
+ assertTrue(applications.get(app1).isPresent());
+ assertEquals(app1, applications.get(app1).get().id());
+ assertTrue(applications.get(app1pr).isPresent());
+ assertEquals(app1pr, applications.get(app1pr).get().id());
+
+ // Simulate restart
+ tester.createNewController();
+ applications = tester.controller().applications();
+
+ assertTrue(applications.get(app1).isPresent());
+ assertEquals(app1, applications.get(app1).get().id());
+ assertTrue(applications.get(app1pr).isPresent());
+ assertEquals(app1pr, applications.get(app1pr).get().id());
+ }
+
+ @Test
+ public void testFailingSinceUpdates() {
+ // Setup system
+ DeploymentTester tester = new DeploymentTester();
+
+ // Setup application
+ Application app = tester.createApplication("app1", "foo", 1, 1L);
+
+ // Initial failure
+ Instant initialFailure = tester.clock().instant();
+ tester.notifyJobCompletion(component, app, true);
+ tester.deployAndNotify(systemTest, app, applicationPackage, false);
+ assertEquals("Failure age is right at initial failure",
+ initialFailure, firstFailing(app, tester).get().at());
+
+ // Failure again -- failingSince should remain the same
+ tester.clock().advance(Duration.ofMillis(1000));
+ tester.deployAndNotify(systemTest, app, applicationPackage, false);
+ assertEquals("Failure age is right at second consecutive failure",
+ initialFailure, firstFailing(app, tester).get().at());
+
+ // Success resets failingSince
+ tester.clock().advance(Duration.ofMillis(1000));
+ tester.deployAndNotify(systemTest, app, applicationPackage, true);
+ assertFalse(firstFailing(app, tester).isPresent());
+
+ // Complete deployment
+ tester.deployAndNotify(stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true);
+
+ // Two repeated failures again.
+ // Initial failure
+ tester.clock().advance(Duration.ofMillis(1000));
+ initialFailure = tester.clock().instant();
+ tester.notifyJobCompletion(component, app, true);
+ tester.deployAndNotify(systemTest, app, applicationPackage, false);
+ assertEquals("Failure age is right at initial failure",
+ initialFailure, firstFailing(app, tester).get().at());
+
+ // Failure again -- failingSince should remain the same
+ tester.clock().advance(Duration.ofMillis(1000));
+ tester.deployAndNotify(systemTest, app, applicationPackage, false);
+ assertEquals("Failure age is right at second consecutive failure",
+ initialFailure, firstFailing(app, tester).get().at());
+ }
+
+ private Optional<JobStatus.JobRun> firstFailing(Application application, DeploymentTester tester) {
+ return tester.controller().applications().get(application.id()).get().deploymentJobs().jobStatus().get(systemTest).firstFailing();
+ }
+
+ @Test
+ public void testMigratingTenantToAthensWillModifyAthensDomainsCorrectly() {
+ ControllerTester tester = new ControllerTester();
+
+ // Create Athens domain mock
+ AthensDomain athensDomain = new AthensDomain("vespa.john");
+ AthensDbMock.Domain mockDomain = new AthensDbMock.Domain(athensDomain);
+ tester.athensDb().addDomain(mockDomain);
+
+ // Create OpsDb tenant
+ TenantId tenantId = new TenantId("mytenant");
+ Tenant existingTenant = Tenant.createOpsDbTenant(tenantId, new UserGroup("myusergroup"), new Property("myproperty"));
+ tester.controller().tenants().addTenant(existingTenant, Optional.empty());
+
+ // Create an application without instance
+ String applicationName = "myapplication";
+ ApplicationId applicationId = ApplicationId.from(tenantId.id(), applicationName, "default");
+ tester.controller().applications().createApplication(applicationId, Optional.empty());
+
+ // Verify that Athens domain does not have any relations to tenant/application yet
+ assertThat(mockDomain.applications.keySet()).isEmpty();
+ assertThat(mockDomain.isVespaTenant).isFalse();
+
+ // Migrate tenant to Athens
+ NToken nToken = new NTokenMock("token");
+ tester.controller().tenants().migrateTenantToAthens(
+ tenantId, athensDomain, new PropertyId("1567"), new Property("vespa_dev.no"), nToken);
+
+ // Verify that tenant is migrated
+ Tenant tenant = tester.controller().tenants().tenant(tenantId).get();
+ assertThat(tenant.isAthensTenant())
+ .isTrue();
+ assertThat(tenant.getAthensDomain().get())
+ .isEqualTo(athensDomain);
+ // Verify that domain knows about tenant and application
+ assertThat(mockDomain.isVespaTenant)
+ .isTrue();
+ assertThat(mockDomain.applications.keySet())
+ .contains(new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationName));
+ }
+
+ @Test
+ public void selfTriggeringApplicationIsNotTriggered() {
+ ControllerTester tester = new ControllerTester();
+ ApplicationController applications = tester.controller().applications();
+
+ // Create application and report completion from component job
+ long projectId = 1;
+ TenantId tenant = tester.createTenant("tenant", "domain", 1L);
+ Application application = tester.createApplication(tenant, "application", "default", projectId);
+ applications.notifyJobCompletion(mockReport(application, component, true, true));
+
+ // Only component completion status is persisted and no further jobs are triggered
+ assertEquals(1, applications.get(application.id()).get().deploymentJobs().jobStatus().size());
+ assertStatus(JobStatus.initial(component).withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()),
+ application.id(), tester.controller());
+ }
+
+ @Test
+ public void requeueOutOfCapacityStagingJob() {
+ DeploymentTester tester = new DeploymentTester();
+
+ long fooProjectId = 1;
+ long barProjectId = 2;
+ Application foo = tester.createApplication("app1", "foo", fooProjectId, 1L);
+ Application bar = tester.createApplication("app2", "bar", barProjectId, 1L);
+ BuildSystem buildSystem = tester.controller().applications().deploymentTrigger().buildSystem();
+
+ // foo: passes system test
+ tester.notifyJobCompletion(component, foo, true);
+ tester.deployAndNotify(systemTest, foo, applicationPackage, true);
+
+ // bar: passes system test
+ tester.notifyJobCompletion(component, bar, true);
+ tester.deployAndNotify(systemTest, bar, applicationPackage, true);
+
+ // foo and bar: staging test jobs queued
+ assertEquals(2, buildSystem.jobs().size());
+
+ // foo: staging-test job fails with out of capacity and is added to the front of the queue
+ {
+ tester.deploy(stagingTest, foo, applicationPackage);
+ tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity));
+ List<BuildJob> nextJobs = buildSystem.takeJobsToRun();
+ assertEquals("staging-test jobs are returned one at a time",1, nextJobs.size());
+ assertEquals(stagingTest.id(), nextJobs.get(0).jobName());
+ assertEquals(fooProjectId, nextJobs.get(0).projectId());
+ }
+
+ // bar: Completes deployment
+ tester.deployAndNotify(stagingTest, bar, applicationPackage, true);
+ tester.deployAndNotify(productionCorpUsEast1, bar, applicationPackage, true);
+
+ // foo: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by
+ // out of capacity retry mechanism
+ tester.clock().advance(Duration.ofMinutes(15));
+ tester.notifyJobCompletion(component, foo, true);
+ tester.deployAndNotify(systemTest, foo, applicationPackage, true);
+ tester.deploy(stagingTest, foo, applicationPackage);
+ assertEquals(1, buildSystem.takeJobsToRun().size());
+ tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity));
+ assertTrue("No jobs queued", buildSystem.jobs().isEmpty());
+
+ // bar: New change triggers another staging-test job
+ tester.notifyJobCompletion(component, bar, true);
+ tester.deployAndNotify(systemTest, bar, applicationPackage, true);
+ assertEquals(1, buildSystem.jobs().size());
+
+ // foo: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the
+ // back of the queue
+ tester.clock().advance(Duration.ofHours(3));
+ tester.clock().advance(Duration.ofMinutes(50));
+ tester.failureRedeployer().maintain();
+
+ List<BuildJob> nextJobs = buildSystem.takeJobsToRun();
+ assertEquals(stagingTest.id(), nextJobs.get(0).jobName());
+ assertEquals(barProjectId, nextJobs.get(0).projectId());
+ nextJobs = buildSystem.takeJobsToRun();
+ assertEquals(stagingTest.id(), nextJobs.get(0).jobName());
+ assertEquals(fooProjectId, nextJobs.get(0).projectId());
+ }
+
+ private void assertStatus(JobStatus expectedStatus, ApplicationId id, Controller controller) {
+ Application app = controller.applications().get(id).get();
+ JobStatus existingStatus = app.deploymentJobs().jobStatus().get(expectedStatus.type());
+ assertNotNull("Status of type " + expectedStatus.type() + " is present", existingStatus);
+ assertEquals(expectedStatus, existingStatus);
+ }
+
+ private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError, boolean selfTriggering) {
+ return new JobReport(
+ application.id(),
+ jobType,
+ application.deploymentJobs().projectId().get(),
+ 1L,
+ jobError,
+ selfTriggering,
+ true
+ );
+ }
+
+ private JobReport mockReport(Application application, JobType jobType, boolean success, boolean selfTriggering) {
+ return mockReport(application, jobType, JobError.from(success), selfTriggering);
+ }
+
+ @Test
+ public void testGlobalRotations() throws IOException {
+ // Setup tester and app def
+ ControllerTester tester = new ControllerTester();
+ Zone zone = Zone.defaultZone();
+ ApplicationId appId = tester.applicationId("tenant", "app1", "default");
+ DeploymentId deployId = new DeploymentId(appId, zone);
+
+ // Check initial rotation status
+ Map<String, EndpointStatus> rotationStatus = tester.controller().applications().getGlobalRotationStatus(deployId);
+ assertEquals(2, rotationStatus.size());
+
+ assertTrue(rotationStatus.get("global-endpoint").getStatus().equals(EndpointStatus.Status.in));
+ assertTrue(rotationStatus.get("alias-endpoint").getStatus().equals(EndpointStatus.Status.in));
+
+ // Set the global rotations out of service
+ EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "Testing I said", "Test", tester.clock().instant().getEpochSecond());
+ List<String> overrides = tester.controller().applications().setGlobalRotationStatus(deployId, status);
+ assertEquals(2, overrides.size());
+
+ // Recheck the override rotation status
+ rotationStatus = tester.controller().applications().getGlobalRotationStatus(deployId);
+ assertEquals(2, rotationStatus.size());
+ assertTrue(rotationStatus.get("global-endpoint").getStatus().equals(EndpointStatus.Status.out));
+ assertTrue(rotationStatus.get("alias-endpoint").getStatus().equals(EndpointStatus.Status.out));
+ assertTrue(rotationStatus.get("alias-endpoint").getReason().equals("Testing I said"));
+ }
+
+ @Test
+ public void testLegacyDeployments() {
+ // Setup system
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationController applications = tester.controller().applications();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+ Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber();
+
+ Application app1 = tester.createApplication("application1", "tenant1", 1, 1L);
+ applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(true)), applications.lock(app1.id()));
+
+ // Scenario: App already on 6.0, Upgrade to 6.1 (systemversion)
+ Zone prodZone = new Zone(Environment.prod, RegionName.from("us-east-3"));
+ Zone stagingZone = new Zone(Environment.staging, RegionName.from("us-east-3"));
+ Version existingVersion = Version.fromString("6.0");
+
+ // Add deployment on existing version
+ legacyDeploy(tester.controller(), app1, applicationPackage, prodZone, Optional.of(existingVersion), false);
+
+ // Add dev/perf deployment on old version to verify that this does not affect Initialize staging step. VESPA-8469
+ Version devVersion = Version.fromString("5.0");
+ legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.dev, RegionName.from("us-east-1")), Optional.of(devVersion), false);
+ legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.perf, RegionName.from("us-east-3")), Optional.of(devVersion), false);
+
+ // Initialize staging on existing version
+ legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), true);
+ app1 = applications.require(app1.id());
+ assertEquals(existingVersion, app1.currentDeployVersion(tester.controller(), stagingZone));
+
+ // Upgrade to the new version in staging
+ legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), false);
+ app1 = applications.require(app1.id());
+ assertEquals(systemVersion, app1.currentDeployVersion(tester.controller(), stagingZone));
+ }
+
+ @Test
+ public void testDeployUntestedChangeFails() {
+ ControllerTester tester = new ControllerTester();
+ ApplicationController applications = tester.controller().applications();TenantId tenant = tester.createTenant("tenant1", "domain1", 11L);
+ Application app = tester.createApplication(tenant, "app1", "default", 1);
+
+ app = app.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.3"))))
+ .with(app.deploymentJobs().asSelfTriggering(false));
+ applications.store(app, applications.lock(app.id()));
+ try {
+ tester.deploy(app, new Zone(Environment.prod, RegionName.from("us-east-3")));
+ fail("Expected exception");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as pending version change to 6.3 is untested", e.getMessage());
+ }
+ }
+
+ private void legacyDeploy(Controller controller, Application application, ApplicationPackage applicationPackage, Zone zone, Optional<Version> version, boolean deployCurrentVersion) {
+ ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get()));
+ GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1"));
+ controller.applications().deployApplication(application.id(),
+ zone,
+ applicationPackage,
+ new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), version, false, deployCurrentVersion));
+
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
new file mode 100644
index 00000000000..41d9bbea5b2
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -0,0 +1,202 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
+import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock;
+import com.yahoo.vespa.hosted.controller.api.integration.jira.JiraMock;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.cost.CostMock;
+import com.yahoo.vespa.hosted.controller.cost.MockInsightBackend;
+import com.yahoo.vespa.hosted.controller.integration.MockMetricsService;
+import com.yahoo.vespa.hosted.controller.persistence.ControllerDb;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import com.yahoo.vespa.hosted.controller.routing.MockRoutingGenerator;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.rotation.MemoryRotationRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock;
+
+import java.util.Optional;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Convenience methods for controller tests.
+ * This completely wraps TestEnvironment to make it easier to get rid of that in the future.
+ *
+ * @author bratseth
+ */
+public final class ControllerTester {
+
+ private final ControllerDb db = new MemoryControllerDb();
+ private final AthensDbMock athensDb = new AthensDbMock();
+ private final ManualClock clock = new ManualClock();
+ private final ConfigServerClientMock configServerClientMock = new ConfigServerClientMock();
+ private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock();
+ private final GitHubMock gitHubMock = new GitHubMock();
+ private final CuratorDb curator = new MockCuratorDb();
+ private Controller controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, athensDb);
+
+ private static final Controller createController(ControllerDb db, CuratorDb curator,
+ ConfigServerClientMock configServerClientMock, ManualClock clock,
+ GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock,
+ AthensDbMock athensDb) {
+ Controller controller = new Controller(db,
+ curator,
+ new MemoryRotationRepository(),
+ gitHubClientMock,
+ new JiraMock(),
+ new MemoryEntityService(),
+ new MemoryGlobalRoutingService(),
+ zoneRegistryMock,
+ new CostMock(new MockInsightBackend()),
+ configServerClientMock,
+ new MockMetricsService(),
+ new MemoryNameService(),
+ new MockRoutingGenerator(),
+ new ChefMock(),
+ clock,
+ new AthensMock(athensDb));
+ controller.updateVersionStatus(VersionStatus.compute(controller));
+ return controller;
+ }
+
+ public Controller controller() { return controller; }
+ public CuratorDb curator() { return curator; }
+ public ManualClock clock() { return clock; }
+ public AthensDbMock athensDb() { return athensDb; }
+
+ /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */
+ public final void createNewController() {
+ controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, athensDb);
+ }
+
+ public ZoneRegistryMock getZoneRegistryMock() { return zoneRegistryMock; }
+
+ public ConfigServerClientMock configServerClientMock() { return configServerClientMock; }
+
+ public GitHubMock gitHubClientMock () { return gitHubMock; }
+
+ /** Set the application with the given id to currently be in the progress of rolling out the given change */
+ public void setDeploying(ApplicationId id, Optional<Change> change) {
+ try (Lock lock = controller.applications().lock(id)) {
+ controller.applications().store(controller.applications().require(id).withDeploying(change), lock);
+ }
+ }
+
+ /** Creates the given tenant and application and deploys it */
+ public Application createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId, Long propertyId) {
+ return createAndDeploy(tenantName, domainName, applicationName, toZone(environment), projectId, propertyId);
+ }
+
+ /** Creates the given tenant and application and deploys it */
+ public Application createAndDeploy(String tenantName, String domainName, String applicationName,
+ String instanceName, Zone zone, long projectId, Long propertyId) {
+ TenantId tenant = createTenant(tenantName, domainName, propertyId);
+ Application application = createApplication(tenant, applicationName, instanceName, projectId);
+ deploy(application, zone);
+ return application;
+ }
+
+ /** Creates the given tenant and application and deploys it */
+ public Application createAndDeploy(String tenantName, String domainName, String applicationName,
+ String instanceName, Environment environment, long projectId, Long propertyId) {
+ return createAndDeploy(tenantName, domainName, applicationName, instanceName, toZone(environment), projectId, propertyId);
+ }
+
+ /** Creates the given tenant and application and deploys it */
+ public Application createAndDeploy(String tenantName, String domainName, String applicationName, Zone zone, long projectId, Long propertyId) {
+ return createAndDeploy(tenantName, domainName, applicationName, "default", zone, projectId, propertyId);
+ }
+
+ /** Creates the given tenant and application and deploys it */
+ public Application createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId) {
+ return createAndDeploy(tenantName, domainName, applicationName, environment, projectId, null);
+ }
+
+ public Zone toZone(Environment environment) {
+ switch (environment) {
+ case dev: case test: return new Zone(environment, RegionName.from("us-east-1"));
+ case staging: return new Zone(environment, RegionName.from("us-east-3"));
+ default: return new Zone(environment, RegionName.from("us-west-1"));
+ }
+ }
+
+ public AthensDomain createDomain(String domainName) {
+ AthensDomain domain = new AthensDomain(domainName);
+ athensDb.addDomain(new AthensDbMock.Domain(domain));
+ return domain;
+ }
+
+ public TenantId createTenant(String tenantName, String domainName, Long propertyId) {
+ TenantId id = new TenantId(tenantName);
+ Optional<Tenant> existing = controller().tenants().tenant(id);
+ if (existing.isPresent()) return id;
+
+ Tenant tenant = Tenant.createAthensTenant(id, createDomain(domainName), new Property("app1Property"),
+ propertyId == null ? Optional.empty() : Optional.of(new PropertyId(propertyId.toString())));
+ controller().tenants().addTenant(tenant, Optional.of(TestIdentities.userNToken));
+ assertNotNull(controller().tenants().tenant(id));
+ return id;
+ }
+
+ public Application createApplication(TenantId tenant, String applicationName, String instanceName, long projectId) {
+ ApplicationId applicationId = applicationId(tenant.id(), applicationName, instanceName);
+ Application application = controller().applications().createApplication(applicationId, Optional.of(TestIdentities.userNToken))
+ .withProjectId(projectId);
+ assertTrue(controller().applications().get(applicationId).isPresent());
+ return application;
+ }
+
+ public void deploy(Application application, Zone zone) {
+ deploy(application, zone, new ApplicationPackage(new byte[0]));
+ }
+
+ public void deploy(Application application, Zone zone, ApplicationPackage applicationPackage) {
+ deploy(application, zone, applicationPackage, false);
+ }
+
+ public void deploy(Application application, Zone zone, ApplicationPackage applicationPackage, boolean deployCurrentVersion) {
+ ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get()));
+ GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1"));
+ controller().applications().deployApplication(application.id(),
+ zone,
+ applicationPackage,
+ new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), Optional.empty(), false, deployCurrentVersion));
+ }
+
+ public ApplicationId applicationId(String tenant, String application, String instance) {
+ return ApplicationId.from(TenantName.from(tenant),
+ ApplicationName.from(application),
+ InstanceName.from(instance));
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java
new file mode 100644
index 00000000000..343a9d2ed6e
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MetricsMock.java
@@ -0,0 +1,83 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+
+import com.yahoo.jdisc.Metric;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class MetricsMock implements Metric {
+
+ private final Map<Context, Map<String, Number>> metrics = new HashMap<>();
+
+ @Override
+ public void set(String key, Number val, Context ctx) {
+ Map<String, Number> metricsMap = metrics.getOrDefault(ctx, new HashMap<>());
+ metricsMap.put(key, val);
+ }
+
+ @Override
+ public void add(String key, Number val, Context ctx) {
+ Map<String, Number> metricsMap = metrics.getOrDefault(ctx, new HashMap<>());
+ metricsMap.compute(key, (k, v) -> v == null ? val : sum(v, val));
+ }
+
+ private Number sum(Number n1, Number n2) {
+ return n1.doubleValue() + n2.doubleValue();
+ }
+
+ @Override
+ public Context createContext(Map<String, ?> properties) {
+ Context ctx = new MapContext(properties);
+ metrics.putIfAbsent(ctx, new HashMap<>());
+ return ctx;
+ }
+
+ public Map<Context, Map<String, Number>> getMetrics() {
+ return metrics;
+ }
+
+ /** Returns a zero-context metric by name, or null if it is not present */
+ public Number getMetric(String name) {
+ Map<String, Number> valuesForEmptyContext = metrics.get(createContext(Collections.emptyMap()));
+ if (valuesForEmptyContext == null) return null;
+ return valuesForEmptyContext.get(name);
+ }
+
+ public Map<MapContext, Map<String, Number>> getMetricsFilteredByHost(String hostname) {
+ return getMetrics().entrySet().stream()
+ .filter(entry -> ((MapContext)entry.getKey()).containsDimensionValue("host", hostname))
+ .collect(Collectors.toMap(entry -> (MapContext) entry.getKey(), Map.Entry::getValue));
+ }
+
+ public static class MapContext implements Context {
+ final Map<String, ?> dimensions;
+
+ public MapContext(Map<String, ?> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return Objects.deepEquals(obj, dimensions);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.toString(dimensions).hashCode();
+ }
+
+ public Map<String, ?> getDimensions() {
+ return dimensions;
+ }
+
+ public boolean containsDimensionValue(String dimension, Object value) {
+ return value.equals(dimensions.get(dimension));
+ }
+ }
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java
new file mode 100644
index 00000000000..1f52ebcadb7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java
@@ -0,0 +1,38 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+
+/**
+ * @author Tony Vaagenes
+ */
+public class TestIdentities {
+
+ public static UserId userId = new UserId("mytenant");
+
+ public static TenantId tenantId = new TenantId("mynonusertenant");
+
+ public static EnvironmentId environment = new EnvironmentId("dev");
+
+ public static RegionId region = new RegionId("us-east-1");
+
+ public static InstanceId instance = new InstanceId("default");
+
+ public static UserGroup userGroup1 = new UserGroup("usergroup1");
+
+ public static Property property = new Property("property");
+
+ public static Tenant tenant = Tenant.createOpsDbTenant(tenantId, userGroup1, property);
+
+ public static NToken userNToken = new NTokenMock("token");
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java
new file mode 100644
index 00000000000..62b935842f7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java
@@ -0,0 +1,74 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author mpolden
+ */
+public class ZoneRegistryMock implements ZoneRegistry {
+
+ private final Map<Zone, Duration> deploymentTimeToLive = new HashMap<>();
+
+ public void setDeploymentTimeToLive(Zone zone, Duration duration) {
+ deploymentTimeToLive.put(zone, duration);
+ }
+
+ @Override
+ public SystemName system() {
+ return SystemName.main;
+ }
+
+ @Override
+ public List<Zone> zones() {
+ return Collections.singletonList(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("corp-us-east-1")));
+ }
+
+ @Override
+ public Optional<Zone> getZone(Environment environment, RegionName region) {
+ return zones().stream().filter(z -> z.environment().equals(environment) && z.region().equals(region)).findFirst();
+ }
+
+ @Override
+ public List<URI> getConfigServerUris(Environment environment, RegionName region) {
+ return getZone(environment, region)
+ .map(z -> URI.create(String.format("http://cfg.%s.%s.test", environment.value(), region.value())))
+ .map(Collections::singletonList)
+ .orElse(Collections.emptyList());
+ }
+
+ @Override
+ public Optional<URI> getLogServerUri(Environment environment, RegionName region) {
+ return getZone(environment, region)
+ .map(z -> URI.create(String.format("http://log.%s.%s.test", environment.value(), region.value())));
+ }
+
+ @Override
+ public Optional<Duration> getDeploymentTimeToLive(Environment environment, RegionName region) {
+ return Optional.ofNullable(deploymentTimeToLive.get(new Zone(environment, region)));
+ }
+
+ @Override
+ public URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application) {
+ return URI.create("http://monitoring-system.test/?environment=" + environment.value() + "&region="
+ + name.value() + "&application=" + application.toShortString());
+ }
+
+ @Override
+ public URI getDashboardUri() {
+ return URI.create("http://dashboard.test");
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java
new file mode 100644
index 00000000000..0a5ddfb5efc
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/CostMock.java
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.cost;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.Backend;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.Cost;
+import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException;
+
+import java.util.List;
+
+/**
+ * @author mpolden
+ */
+public class CostMock implements Cost {
+
+ private final Backend backend;
+
+ public CostMock(Backend backend) {
+ this.backend = backend;
+ }
+
+ @Override
+ public List<ApplicationCost> getCPUAnalysis(int nofApplications) {
+ return null;
+ }
+
+ @Override
+ public String getCsvForLocalAnalysis() {
+ return null;
+ }
+
+ @Override
+ public List<ApplicationCost> getApplicationCost() {
+ return backend.getApplicationCost();
+ }
+
+ @Override
+ public ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId app) throws NotFoundCheckedException {
+ return backend.getApplicationCost(env, region, app);
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java
new file mode 100644
index 00000000000..c4ba5fa4fc5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/cost/MockInsightBackend.java
@@ -0,0 +1,41 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.cost;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.Backend;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author bratseth
+ */
+public class MockInsightBackend extends AbstractComponent implements Backend {
+
+ private final Map<ApplicationId, ApplicationCost> applicationCost = new HashMap<>();
+
+ @Override
+ public List<ApplicationCost> getApplicationCost() {
+ return new ArrayList<>(applicationCost.values());
+ }
+
+ /**
+ * Get cost for a specific application in one zone or null if this application is not known.
+ * The zone information is ignored in the dummy backend.
+ */
+ @Override
+ public ApplicationCost getApplicationCost(Environment env, RegionName region, ApplicationId application) {
+ return applicationCost.get(application);
+ }
+
+ public void setApplicationCost(ApplicationId application, ApplicationCost cost) {
+ applicationCost.put(application, cost);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
new file mode 100644
index 00000000000..aa115421f6a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -0,0 +1,113 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * A builder that builds application packages for testing purposes.
+ *
+ * @author mpolden
+ */
+public class ApplicationPackageBuilder {
+
+ private String upgradePolicy = null;
+ private Environment environment = Environment.prod;
+ private final StringBuilder environmentBody = new StringBuilder();
+ private final StringBuilder validationOverridesBody = new StringBuilder();
+
+ public ApplicationPackageBuilder upgradePolicy(String upgradePolicy) {
+ this.upgradePolicy = upgradePolicy;
+ return this;
+ }
+
+ public ApplicationPackageBuilder environment(Environment environment) {
+ this.environment = environment;
+ return this;
+ }
+
+ public ApplicationPackageBuilder region(String regionName) {
+ environmentBody.append(" <region active='true'>");
+ environmentBody.append(regionName);
+ environmentBody.append("</region>\n");
+ return this;
+ }
+
+ public ApplicationPackageBuilder delay(Duration delay) {
+ environmentBody.append(" <delay seconds='");
+ environmentBody.append(delay.getSeconds());
+ environmentBody.append("'/>\n");
+ return this;
+ }
+
+ public ApplicationPackageBuilder allow(ValidationId validationId) {
+ validationOverridesBody.append(" <allow until='");
+ validationOverridesBody.append(asIso8601String(Instant.now().plus(Duration.ofDays(29))));
+ validationOverridesBody.append("'>");
+ validationOverridesBody.append(validationId.value());
+ validationOverridesBody.append("</allow>\n");
+ return this;
+ }
+
+ private byte[] deploymentSpec() {
+ StringBuilder xml = new StringBuilder("<deployment version='1.0'>\n");
+ if (upgradePolicy != null) {
+ xml.append("<upgrade policy='");
+ xml.append(upgradePolicy);
+ xml.append("'/>\n");
+ }
+ xml.append(" <");
+ xml.append(environment.value());
+ xml.append(">\n");
+ xml.append(environmentBody);
+ xml.append(" </");
+ xml.append(environment.value());
+ xml.append(">\n</deployment>");
+ return xml.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ private byte[] validationOverrides() {
+ String xml = "<validation-overrides version='1.0'>\n" +
+ validationOverridesBody +
+ "</validation-overrides>\n";
+ return xml.getBytes(StandardCharsets.UTF_8);
+ }
+
+ public ApplicationPackage build() {
+ ByteArrayOutputStream zip = new ByteArrayOutputStream();
+ ZipOutputStream out = new ZipOutputStream(zip);
+ try {
+ out.putNextEntry(new ZipEntry("deployment.xml"));
+ out.write(deploymentSpec());
+ out.closeEntry();
+ out.putNextEntry(new ZipEntry("validation-overrides.xml"));
+ out.write(validationOverrides());
+ out.closeEntry();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ } finally {
+ try {
+ out.close();
+ } catch (IOException ignored) {}
+ }
+ return new ApplicationPackage(zip.toByteArray());
+ }
+
+ private static String asIso8601String(Instant instant) {
+ DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE.withZone(ZoneId.systemDefault() );
+ return formatter.format(instant);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
new file mode 100644
index 00000000000..32d1714ea52
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -0,0 +1,209 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.ConfigServerClientMock;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.maintenance.FailureRedeployer;
+import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
+import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class DeploymentTester {
+
+ private ControllerTester tester = new ControllerTester();
+
+ private Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(2),
+ new JobControl(tester.curator()));
+ private FailureRedeployer failureRedeployer = new FailureRedeployer(tester.controller(), Duration.ofMinutes(2),
+ new JobControl(tester.curator()));
+
+ public Upgrader upgrader() { return upgrader; }
+ public FailureRedeployer failureRedeployer() { return failureRedeployer; }
+ public Controller controller() { return tester.controller(); }
+ public ApplicationController applications() { return tester.controller().applications(); }
+ public BuildSystem buildSystem() { return tester.controller().applications().deploymentTrigger().buildSystem(); }
+ public DeploymentTrigger deploymentTrigger() { return tester.controller().applications().deploymentTrigger(); }
+ public ManualClock clock() { return tester.clock(); }
+ public ControllerTester controllerTester() { return tester; }
+
+ public Application application(String name) {
+ return application(ApplicationId.from("tenant1", name, "default"));
+ }
+
+ public Application application(ApplicationId application) {
+ return controller().applications().require(application);
+ }
+
+ public Optional<Change.VersionChange> versionChange(ApplicationId application) {
+ return application(application).deploying()
+ .filter(c -> c instanceof Change.VersionChange)
+ .map(Change.VersionChange.class::cast);
+ }
+
+ public ConfigServerClientMock configServerClientMock() { return tester.configServerClientMock(); }
+
+ public void updateVersionStatus(Version currentVersion) {
+ controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion));
+ }
+
+ public void upgradeSystem(Version version) {
+ updateVersionStatus(version);
+ upgrader().maintain();
+ }
+
+ public Application createApplication(String applicationName, String tenantName, long projectId, Long propertyId) {
+ TenantId tenant = tester.createTenant(tenantName, UUID.randomUUID().toString(), propertyId);
+ return tester.createApplication(tenant, applicationName, "default", projectId);
+ }
+
+ public void restartController() { tester.createNewController(); }
+
+ /** Simulate the full lifecycle of an application deployment as declared in given application package */
+ public Application createAndDeploy(String applicationName, int projectId, ApplicationPackage applicationPackage) {
+ tester.createTenant("tenant1", "domain1", 1L);
+ Application application = tester.createApplication(new TenantId("tenant1"), applicationName, "default", projectId);
+ deployCompletely(application, applicationPackage);
+ return applications().require(application.id());
+ }
+
+ /** Simulate the full lifecycle of an application deployment to prod.us-west-1 with the given upgrade policy */
+ public Application createAndDeploy(String applicationName, int projectId, String upgradePolicy) {
+ return createAndDeploy(applicationName, projectId, applicationPackage(upgradePolicy));
+ }
+
+ /** Complete an ongoing deployment */
+ public void deployCompletely(String applicationName) {
+ deployCompletely(applications().require(ApplicationId.from("tenant1", applicationName, "default")),
+ applicationPackage("default"));
+ }
+
+ /** Deploy application completely using the given application package */
+ public void deployCompletely(Application application, ApplicationPackage applicationPackage) {
+ notifyJobCompletion(JobType.component, application, true);
+ assertTrue(applications().require(application.id()).deploying().isPresent());
+ completeDeployment(application, applicationPackage, Optional.empty());
+ }
+
+ private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) {
+ List<JobType> triggerOrder = JobType.triggerOrder(SystemName.main, applicationPackage.deploymentSpec());
+ for (JobType job : triggerOrder) {
+ boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false);
+ deployAndNotify(job, application, applicationPackage, !failJob);
+ if (failJob) {
+ break;
+ }
+ }
+ if (failOnJob.isPresent()) {
+ assertTrue(applications().require(application.id()).deploying().isPresent());
+ assertTrue(applications().require(application.id()).deploymentJobs().hasFailures());
+ } else {
+ assertFalse(applications().require(application.id()).deploying().isPresent());
+ }
+ }
+
+ public void notifyJobCompletion(JobType jobType, Application application, boolean success) {
+ notifyJobCompletion(jobType, application, DeploymentJobs.JobError.from(success));
+ }
+
+ public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError) {
+ applications().notifyJobCompletion(jobReport(application, jobType, jobError));
+ }
+
+ public void completeUpgrade(Application application, Version version, String upgradePolicy) {
+ assertTrue(applications().require(application.id()).deploying().isPresent());
+ assertEquals(new Change.VersionChange(version), applications().require(application.id()).deploying().get());
+ completeDeployment(application, applicationPackage(upgradePolicy), Optional.empty());
+ }
+
+ public void completeUpgradeWithError(Application application, Version version, String upgradePolicy, JobType failOnJob) {
+ completeUpgradeWithError(application, version, applicationPackage(upgradePolicy), Optional.of(failOnJob));
+ }
+
+ public void completeUpgradeWithError(Application application, Version version, ApplicationPackage applicationPackage, JobType failOnJob) {
+ completeUpgradeWithError(application, version, applicationPackage, Optional.of(failOnJob));
+ }
+
+ private void completeUpgradeWithError(Application application, Version version, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) {
+ assertTrue(applications().require(application.id()).deploying().isPresent());
+ assertEquals(new Change.VersionChange(version), applications().require(application.id()).deploying().get());
+ completeDeployment(application, applicationPackage, failOnJob);
+ }
+
+ public void deploy(JobType job, Application application, ApplicationPackage applicationPackage) {
+ deploy(job, application, applicationPackage, false);
+ }
+
+ public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) {
+ job.zone(SystemName.main).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion));
+ }
+
+ public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) {
+ assertScheduledJob(application, job);
+ if (success) {
+ deploy(job, application, applicationPackage);
+ }
+ notifyJobCompletion(job, application, success);
+ }
+
+ private void assertScheduledJob(Application application, JobType jobType) {
+ Optional<BuildService.BuildJob> job = findJob(application, jobType);
+ assertTrue(String.format("Job %s is scheduled for %s", jobType, application), job.isPresent());
+ buildSystem().removeJobs(application.id());
+ assertEquals((long) application.deploymentJobs().projectId().get(), job.get().projectId());
+ assertEquals(jobType.id(), job.get().jobName());
+ }
+
+ private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) {
+ for (BuildService.BuildJob job : buildSystem().jobs())
+ if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id()))
+ return Optional.of(job);
+ return Optional.empty();
+ }
+
+ private DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) {
+ return new DeploymentJobs.JobReport(
+ application.id(),
+ jobType,
+ application.deploymentJobs().projectId().get(),
+ 1L,
+ jobError,
+ false,
+ true
+ );
+ }
+
+ private static ApplicationPackage applicationPackage(String upgradePolicy) {
+ return new ApplicationPackageBuilder()
+ .upgradePolicy(upgradePolicy)
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .build();
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
new file mode 100644
index 00000000000..ce06910240b
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -0,0 +1,167 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ * @author mpolden
+ */
+public class DeploymentTriggerTest {
+
+ @Test
+ public void testTriggerFailing() {
+ DeploymentTester tester = new DeploymentTester();
+ Application app1 = tester.createAndDeploy("app1", 1, "default");
+
+ Version version = new Version(5, 2);
+ tester.deploymentTrigger().triggerChange(app1.id(), new Change.VersionChange(version));
+ tester.completeUpgradeWithError(app1, version, "default", JobType.stagingTest);
+ assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size());
+
+ tester.buildSystem().takeJobsToRun();
+ assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
+ tester.clock().advance(Duration.ofHours(2));
+ tester.deploymentTrigger().triggerFailing(app1.id());
+ assertEquals("Retried job", 1, tester.buildSystem().jobs().size());
+ assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName());
+
+ tester.buildSystem().takeJobsToRun();
+ assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
+ tester.clock().advance(Duration.ofHours(7));
+ tester.deploymentTrigger().triggerFailing(app1.id());
+ assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size());
+ assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName());
+ }
+
+ @Test
+ public void deploymentSpecDecidesTriggerOrder() {
+ DeploymentTester tester = new DeploymentTester();
+ BuildSystem buildSystem = tester.buildSystem();
+ TenantId tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L);
+ Application application = tester.controllerTester().createApplication(tenant, "app1", "default", 1L);
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .region("us-central-1")
+ .region("us-west-1")
+ .build();
+
+ // Component job finishes
+ tester.notifyJobCompletion(JobType.component, application, true);
+
+ // Application is deployed to all test environments and declared zones
+ tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true);
+ tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true);
+ tester.deployAndNotify(JobType.productionCorpUsEast1, application, applicationPackage, true);
+ tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true);
+ tester.deployAndNotify(JobType.productionUsWest1, application, applicationPackage, true);
+ assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
+ }
+
+ @Test
+ public void deploymentsSpecWithDelays() {
+ DeploymentTester tester = new DeploymentTester();
+ BuildSystem buildSystem = tester.buildSystem();
+ Application application = tester.createApplication("app1", "tenant1", 1, 1L);
+
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .delay(Duration.ofSeconds(30))
+ .region("us-west-1")
+ .delay(Duration.ofMinutes(1))
+ .delay(Duration.ofMinutes(2)) // Multiple delays are summed up
+ .region("us-central-1")
+ .delay(Duration.ofMinutes(10)) // Delays after last region are valid, but have no effect
+ .build();
+
+ // Component job finishes
+ tester.notifyJobCompletion(JobType.component, application, true);
+
+ // Test jobs pass
+ tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true);
+ tester.clock().advance(Duration.ofSeconds(1)); // Make staging test sort as the last successful job
+ tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true);
+ assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty());
+
+ // 30 seconds pass, us-west-1 is triggered
+ tester.clock().advance(Duration.ofSeconds(30));
+ tester.deploymentTrigger().triggerDelayed();
+
+ // Consume us-west-1 job without reporting completion
+ assertEquals(1, buildSystem.jobs().size());
+ assertEquals(JobType.productionUsWest1.id(), buildSystem.jobs().get(0).jobName());
+ buildSystem.takeJobsToRun();
+
+ // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress
+ tester.clock().advance(Duration.ofMinutes(3));
+ tester.deploymentTrigger().triggerDelayed();
+ assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty());
+
+ // us-west-1 completes
+ tester.deploy(JobType.productionUsWest1, application, applicationPackage);
+ tester.notifyJobCompletion(JobType.productionUsWest1, application, true);
+
+ // Delayed trigger does nothing as not enough time has passed after us-west-1 completion
+ tester.deploymentTrigger().triggerDelayed();
+ assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty());
+
+ // 3 minutes pass, us-central-1 is triggered
+ tester.clock().advance(Duration.ofMinutes(3));
+ tester.deploymentTrigger().triggerDelayed();
+ tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true);
+ assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
+
+ // Delayed trigger job runs again, with nothing to trigger
+ tester.clock().advance(Duration.ofMinutes(10));
+ tester.deploymentTrigger().triggerDelayed();
+ assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
+ }
+
+
+ @Test
+ public void testSuccessfulDeploymentApplicationPackageChanged() {
+ DeploymentTester tester = new DeploymentTester();
+ BuildSystem buildSystem = tester.buildSystem();
+ TenantId tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L);
+ Application application = tester.controllerTester().createApplication(tenant, "app1", "default", 1L);
+ ApplicationPackage previousApplicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .region("us-central-1")
+ .region("us-west-1")
+ .build();
+ ApplicationPackage newApplicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .region("us-central-1")
+ .region("us-west-1")
+ .region("ap-northeast-1")
+ .build();
+
+ // Component job finishes
+ tester.notifyJobCompletion(JobType.component, application, true);
+
+ // Application is deployed to all test environments and declared zones
+ tester.deployAndNotify(JobType.systemTest, application, newApplicationPackage, true);
+ tester.deploy(JobType.stagingTest, application, previousApplicationPackage, true);
+ tester.deployAndNotify(JobType.stagingTest, application, newApplicationPackage, true);
+ tester.deployAndNotify(JobType.productionCorpUsEast1, application, newApplicationPackage, true);
+ tester.deployAndNotify(JobType.productionUsCentral1, application, newApplicationPackage, true);
+ tester.deployAndNotify(JobType.productionUsWest1, application, newApplicationPackage, true);
+ tester.deployAndNotify(JobType.productionApNortheast1, application, newApplicationPackage, true);
+ assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java
new file mode 100644
index 00000000000..6346d1cbdb6
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java
@@ -0,0 +1,179 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+
+import java.time.Duration;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.QUEUED;
+import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.RUNNING;
+
+/**
+ * Simulates polling of build jobs from the controller and triggering and execution of
+ * these in Screwdriver.
+ *
+ * @author jvenstad
+ */
+public class MockBuildService implements BuildService {
+
+ private final ControllerTester tester;
+ private final MockTimeline timeline;
+ private final Map<String, Job> jobs;
+ private final Map<String, JobStatus> jobStatuses;
+ private Version version;
+
+ public MockBuildService(ControllerTester tester, MockTimeline timeline) {
+ this.tester = tester;
+ this.timeline = timeline;
+ jobs = new HashMap<>();
+ jobStatuses = new HashMap<>();
+ version = new Version(6, 86);
+ }
+
+ /** Simulates the triggering of a Screwdriver job, where jobs are queued if already running. */
+ @Override
+ public boolean trigger(BuildJob buildJob) {
+ String key = buildJob.toString();
+ System.err.println(timeline.now() + ": Asked to trigger " + key);
+
+ if ( ! jobStatuses.containsKey(key))
+ startJob(key);
+ else
+ jobStatuses.put(key, QUEUED);
+
+ return true;
+ }
+
+ /** Simulates the internal triggering of Screwdriver, where only one instance is run at a time. */
+ private void startJob(String key) {
+ jobStatuses.put(key, RUNNING);
+ Job job = jobs.get(key);
+ if (job == null)
+ return;
+
+ timeline.in(job.duration, () -> {
+ job.outcome();
+ if (jobStatuses.get(key) == QUEUED)
+ startJob(key);
+ else
+ jobStatuses.remove(key);
+ });
+ System.err.println(timeline.now() + ": Triggered " + key + "; it will finish at " + timeline.now().plus(job.duration));
+ }
+
+ public void incrementVersion() {
+ version = new Version(version.getMajor(), version.getMinor() + 1);
+ }
+
+ public Version version() { return version; }
+
+ /** Add @job to the set of @Job objects we have information about. */
+ private void add(Job job) {
+ jobs.put(job.buildJob().toString(), job);
+ }
+
+ /** Add @project to the set of @Project objects we have information about. */
+ private void add(Project project) {
+ project.jobs.values().forEach(this::add);
+ }
+
+ /** Make a @Project with the given settings, modify it if desired, and @add() it its jobs to the pool of known ones. */
+ public Project project(ApplicationId applicationId, Long projectId, Duration duration, Supplier<Boolean> success) {
+ return new Project(applicationId, projectId, duration, success);
+ }
+
+
+ /** Convenience creator for many jobs, belonging to the same project. Jobs can be modified independently after creation. */
+ class Project {
+
+ private final ApplicationId applicationId;
+ private final Long projectId;
+ private final Duration duration;
+ private final Supplier<Boolean> success;
+ private final Map<JobType, Job> jobs;
+
+ private Project(ApplicationId applicationId, Long projectId, Duration duration, Supplier<Boolean> success) {
+ this.applicationId = applicationId;
+ this.projectId = projectId;
+ this.duration = duration;
+ this.success = success;
+
+ jobs = new EnumMap<>(JobType.class);
+
+ for (JobType jobType : JobType.values())
+ jobs.put(jobType, new Job(applicationId, projectId, jobType, duration, success));
+ }
+
+ /** Set @duration for @jobType of this @Project. */
+ public Project set(Duration duration, JobType jobType) {
+ jobs.compute(jobType, (type, job) -> new Job(applicationId, projectId, jobType, duration, job.success));
+ return this;
+ }
+
+ /** Set @success for @jobType of this @Project. */
+ public Project set(Supplier<Boolean> success, JobType jobType) {
+ jobs.compute(jobType, (type, job) -> new Job(applicationId, projectId, jobType, job.duration, success));
+ return this;
+ }
+
+ /** Add the @Job objects of this @Project to the pool of known jobs for this @MockBuildService. */
+ public void add() {
+ MockBuildService.this.add(this);
+ }
+
+ }
+
+
+ /** Representation of a simulated job -- most noteworthy is the @outcome(), which is used to simulate a job completing. */
+ private class Job {
+
+ private final ApplicationId applicationId;
+ private final Long projectId;
+ private final JobType jobType;
+ private final Duration duration;
+ private final Supplier<Boolean> success;
+
+ private Job(ApplicationId applicationId, Long projectId, JobType jobType, Duration duration, Supplier<Boolean> success) {
+ this.applicationId = applicationId;
+ this.projectId = projectId;
+ this.jobType = jobType;
+ this.duration = duration;
+ this.success = success;
+ }
+
+ private void outcome() {
+ Boolean success = this.success.get();
+ System.err.println(timeline.now() + ": Job " + projectId + ":" + jobType + " reports " + success);
+ if (success != null)
+ tester.controller().applications().notifyJobCompletion(
+ new DeploymentJobs.JobReport(
+ applicationId,
+ jobType,
+ projectId,
+ 1L,
+ JobError.from(success),
+ false,
+ false
+ ));
+ }
+
+ private BuildJob buildJob() { return new BuildJob(projectId, jobType.id()); }
+
+ }
+
+ enum JobStatus {
+ QUEUED,
+ RUNNING
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java
new file mode 100644
index 00000000000..878c25bf6bd
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockTimeline.java
@@ -0,0 +1,106 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.test.ManualClock;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.PriorityQueue;
+
+/**
+ * @author jvenstad
+ */
+public class MockTimeline {
+
+ private final ManualClock clock;
+ private final PriorityQueue<Event> events;
+
+ public MockTimeline(ManualClock clock) {
+ this.events = new PriorityQueue<>();
+ this.clock = clock;
+ }
+
+ /** Make @event happen at time @at, as measured by the internal clock. */
+ public void at(Instant at, Runnable event) {
+ if (at.isBefore(now()))
+ throw new IllegalArgumentException("The flow of time runs only one way, my friend.");
+ events.add(new Event(at, event));
+ }
+
+ /** Make @event happen in @in time, as measured by the internal clock. */
+ public void in(Duration in, Runnable event) {
+ at(now().plus(in), event);
+ }
+
+ /** Make @event happen every @period time, starting @offset time from @now(), as measured by the internal clock. */
+ public void every(Duration period, Duration offset, Runnable event) {
+ in(offset, () -> {
+ every(period, event);
+ event.run();
+ });
+ }
+
+ /** Make @event happen every @period time, starting @period time from @now(), as measured by the internal clock. */
+ public void every(Duration period, Runnable event) {
+ every(period, period, event);
+ }
+
+ /** Returns the current time, as measured by the internal clock. */
+ public Instant now() {
+ return clock.instant();
+ }
+
+ /** Returns whether there are more events in the timeline, or not. */
+ public boolean hasNext() {
+ return ! events.isEmpty();
+ }
+
+ /** Advance time to the next event, let it happen, and return the time of this event. */
+ public Instant next() {
+ Event event = events.poll();
+ clock.advance(Duration.ofMillis(now().until(event.at(), ChronoUnit.MILLIS)));
+ event.happen();
+ return event.at();
+ }
+
+ /** Advance the time until @until, letting all events from now to then happen. */
+ public void advance(Instant until) {
+ at(until, () -> {});
+ while (next() != until);
+ }
+
+ /** Advance the time by @duration, letting all events from now to then happen. */
+ public void advance(Duration duration) {
+ advance(now().plus(duration));
+ }
+
+ /** Let the timeline unfold! Careful about those @every-s, though... */
+ public void unfold() {
+ while (hasNext())
+ next();
+ }
+
+
+ private static class Event implements Comparable<Event> {
+
+ private final Instant at;
+ private final Runnable event;
+
+ private Event(Instant at, Runnable event) {
+ this.at = at;
+ this.event = event;
+ }
+
+ public Instant at() { return at; }
+ public void happen() { event.run(); }
+
+
+ @Override
+ public int compareTo(Event other) {
+ return at().compareTo(other.at());
+ }
+
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java
new file mode 100644
index 00000000000..779af370ff4
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java
@@ -0,0 +1,64 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+@RunWith(Parameterized.class)
+public class PolledBuildSystemTest {
+
+ @Parameterized.Parameters(name = "jobType={0}")
+ public static Iterable<? extends Object> capacityConstrainedJobs() {
+ return Arrays.asList(JobType.systemTest, JobType.stagingTest);
+ }
+
+ private final JobType jobType;
+
+ public PolledBuildSystemTest(JobType jobType) {
+ this.jobType = jobType;
+ }
+
+ @Test
+ public void throttle_capacity_constrained_jobs() {
+ ControllerTester tester = new ControllerTester();
+ BuildSystem buildSystem = new PolledBuildSystem(tester.controller(), new MockCuratorDb());
+
+ long fooProjectId = 1;
+ long barProjectId = 2;
+ ApplicationId foo = tester.createAndDeploy("tenant1", "domain1", "app1",
+ Environment.prod, fooProjectId).id();
+ ApplicationId bar = tester.createAndDeploy("tenant2", "domain2", "app2",
+ Environment.prod, barProjectId).id();
+
+ // Trigger jobs in capacity constrained environment
+ buildSystem.addJob(foo, jobType, false);
+ buildSystem.addJob(bar, jobType, false);
+
+ // Capacity constrained jobs are returned one a at a time
+ List<BuildJob> nextJobs = buildSystem.takeJobsToRun();
+ assertEquals(1, nextJobs.size());
+ assertEquals(fooProjectId, nextJobs.get(0).projectId());
+
+ nextJobs = buildSystem.takeJobsToRun();
+ assertEquals(1, nextJobs.size());
+ assertEquals(barProjectId, nextJobs.get(0).projectId());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java
new file mode 100644
index 00000000000..360fd8616d3
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java
@@ -0,0 +1,22 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.integration;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+
+/**
+ * @author bratseth
+ */
+public class MockMetricsService implements com.yahoo.vespa.hosted.controller.api.integration.MetricsService {
+
+ @Override
+ public ApplicationMetrics getApplicationMetrics(ApplicationId application) {
+ return new ApplicationMetrics(0.5, 0.7);
+ }
+
+ @Override
+ public DeploymentMetrics getDeploymentMetrics(ApplicationId application, Zone zone) {
+ return new DeploymentMetrics(1, 2, 3, 4, 5);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
new file mode 100644
index 00000000000..4c53a6d37e4
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
@@ -0,0 +1,46 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class DeploymentExpirerTest {
+
+ @Test
+ public void testDeploymentExpiry() throws IOException, InterruptedException {
+ ControllerTester tester = new ControllerTester();
+ tester.getZoneRegistryMock().setDeploymentTimeToLive(new Zone(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14));
+ DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10),
+ tester.clock(), new JobControl(new MockCuratorDb()));
+ ApplicationId devApp = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id();
+ ApplicationId prodApp = tester.createAndDeploy("tenant2", "domain2", "app2", Environment.prod, 456).id();
+
+ assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size());
+ assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size());
+
+ // Not expired at first
+ expirer.maintain();
+ assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size());
+ assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size());
+
+ // The dev application is removed
+ tester.clock().advance(Duration.ofDays(15));
+ expirer.maintain();
+ assertEquals(0, tester.controller().applications().get(devApp).get().deployments().size());
+ assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
new file mode 100644
index 00000000000..f5a76f6446c
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
@@ -0,0 +1,280 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.integration.Contacts.UserContact;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues;
+import com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo;
+import com.yahoo.vespa.hosted.controller.api.integration.stubs.ContactsMock;
+import com.yahoo.vespa.hosted.controller.api.integration.stubs.PropertiesMock;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.admin;
+import static com.yahoo.vespa.hosted.controller.api.integration.Contacts.Category.engineeringOwner;
+import static com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo.Status.done;
+import static com.yahoo.vespa.hosted.controller.api.integration.Issues.IssueInfo.Status.toDo;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
+import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxFailureAge;
+import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxInactivityAge;
+import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.terminalUser;
+import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.vespaOps;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author jvenstad
+ */
+public class DeploymentIssueReporterTest {
+
+ private final static ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+
+ private DeploymentTester tester;
+ private DeploymentIssueReporter reporter;
+ private ContactsMock contacts;
+ private PropertiesMock properties;
+ private MockIssues issues;
+
+ @Before
+ public void setup() {
+ tester = new DeploymentTester();
+ contacts = new ContactsMock();
+ properties = new PropertiesMock();
+ issues = new MockIssues(tester.clock());
+ reporter = new DeploymentIssueReporter(tester.controller(), contacts, properties, issues, Duration.ofMinutes(5),
+ new JobControl(new MockCuratorDb()));
+ }
+
+ private List<IssueInfo> openIssuesFor(Application application) {
+ return issues.fetchSimilarTo(reporter.deploymentIssueFrom(tester.controller().applications().require(application.id())));
+ }
+
+ @Test
+ public void testDeploymentFailureReporting() {
+ // All applications deploy from unique SD projects.
+ Long projectId1 = 10L;
+ Long projectId2 = 20L;
+ Long projectId3 = 30L;
+
+ // Only the first two have propertyIds set now.
+ Long propertyId1 = 1L;
+ Long propertyId2 = 2L;
+
+ // Create and deploy one application for each of three tenants.
+ Application app1 = tester.createApplication("application1", "tenant1", projectId1, propertyId1);
+ Application app2 = tester.createApplication("application2", "tenant2", projectId2, propertyId2);
+ Application app3 = tester.createApplication("application3", "tenant3", projectId3, null);
+
+ // And then we need lots of successful applications, so we won't assume we just have a faulty Vespa out.
+ for (long i = 4; i <= 10; i++) {
+ Application app = tester.createApplication("application" + i, "tenant" + i, 10 * i, i);
+ tester.notifyJobCompletion(component, app, true);
+ tester.deployAndNotify(systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true);
+ }
+
+ // Both the first tenants belong to the same JIRA queue. (Not sure if this is possible, but let's test it anyway.
+ String jiraQueue = "PROJECT";
+ properties.addClassification(propertyId1, jiraQueue);
+ properties.addClassification(propertyId1, jiraQueue);
+
+ // Only tenant1 has contacts listed in opsDb.
+ UserContact
+ alice = new UserContact("alice", "Alice", admin),
+ bob = new UserContact("bob", "Robert", engineeringOwner);
+ contacts.addContact(propertyId1, Arrays.asList(alice, bob));
+
+ // end of setup.
+
+ // NOTE: All maintenance should be idempotent within a small enough time interval, so maintain is called twice in succession throughout.
+
+ // app1 and app3 has one failure each.
+ tester.notifyJobCompletion(component, app1, true);
+ tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app1, applicationPackage, false);
+
+ tester.notifyJobCompletion(component, app2, true);
+ tester.deployAndNotify(systemTest, app2, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app2, applicationPackage, true);
+
+ tester.notifyJobCompletion(component, app3, true);
+ tester.deployAndNotify(systemTest, app3, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app3, applicationPackage, true);
+ tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false);
+
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("No deployments are detected as failing for a long time initially.", 0, issues.issues.size());
+
+
+ // Advance to where deployment issues should be detected.
+ tester.clock().advance(maxFailureAge.plus(Duration.ofDays(1)));
+
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("One issue is produced for app1.", 1, openIssuesFor(app1).size());
+ assertEquals("No issues are produced for app2.", 0, openIssuesFor(app2).size());
+ assertEquals("One issue is produced for app3.", 1, openIssuesFor(app3).size());
+ assertTrue("The issue for app1 is stored in their JIRA queue.", openIssuesFor(app1).get(0).key().startsWith(jiraQueue));
+ assertTrue("The issue for an application without propertyId is addressed to vespaOps.", openIssuesFor(app3).get(0).key().startsWith(vespaOps.queue()));
+
+
+ // Verify idempotency of filing.
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("No issues are re-filed when still open.", 2, issues.issues.size());
+
+
+ // tenant3 closes their issue prematurely; see that we get a new filing.
+ issues.complete(openIssuesFor(app3).get(0).id());
+ assertEquals("The issue is removed (test of the tester, really...).", 0, openIssuesFor(app3).size());
+
+ reporter.maintain();
+ reporter.maintain();
+ assertTrue("Issue is re-produced for app3, addressed correctly.", openIssuesFor(app3).get(0).key().startsWith(vespaOps.queue()));
+
+
+ // Some time passes; tenant1 leaves her issue unattended, while tenant3 starts work and updates the issue.
+ // app2 also has an intermittent failure; see that we detect this as a Vespa problem, and file an issue to ourselves.
+ tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, false);
+ tester.clock().advance(maxInactivityAge.plus(maxFailureAge));
+ issues.comment(openIssuesFor(app3).get(0).id(), "We are trying to fix it!");
+
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("The issue for app1 is escalated once.", alice.username(), openIssuesFor(app1).get(0).assignee().get());
+
+
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("We get an issue to vespaOps when more than 20% of applications have old failures.", 1,
+ issues.fetchSimilarTo(reporter.manyFailingDeploymentsIssueFrom(Arrays.asList(
+ tester.controller().applications().get(app1.id()).get(),
+ tester.controller().applications().get(app2.id()).get(),
+ tester.controller().applications().get(app3.id()).get()))).size());
+ assertEquals("No issue is filed for app2 while Vespa is considered broken.", 0, openIssuesFor(app2).size());
+
+
+ // app3 fixes its problem, but the ticket is left open; see the resolved ticket is not escalated when another escalation period has passed.
+ tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, true);
+ tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, true);
+ tester.clock().advance(maxInactivityAge.plus(Duration.ofDays(1)));
+
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("The issue for app1 is escalated once more.", bob.username(), openIssuesFor(app1).get(0).assignee().get());
+ assertEquals("The issue for app3 is still unassigned.", Optional.empty(), openIssuesFor(app3).get(0).assignee());
+
+
+ // app1 still does nothing with their issue; see the terminal user gets it in the end.
+ // app3 now has a new failure past max failure age; see that a new issue is filed.
+ tester.notifyJobCompletion(component, app3, true);
+ tester.deployAndNotify(systemTest, app3, applicationPackage, true);
+ tester.deployAndNotify(stagingTest, app3, applicationPackage, true);
+ tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false);
+ tester.clock().advance(maxInactivityAge.plus(maxFailureAge));
+
+ reporter.maintain();
+ reporter.maintain();
+ assertEquals("The issue for app1 is escalated to the terminal user.", terminalUser.username(), openIssuesFor(app1).get(0).assignee().get());
+ assertEquals("A new issue is filed for app3.", 2, openIssuesFor(app3).size());
+ }
+
+ class MockIssues implements Issues {
+
+ final Map<String, Issue> issues = new HashMap<>();
+ final Map<String, IssueInfo> metas = new HashMap<>();
+ final Map<String, Long> counters = new HashMap<>();
+ Clock clock;
+
+ MockIssues(Clock clock) { this.clock = clock; }
+
+ public void addWatcher(String jiraIssueId, String watcher) {
+ touch(jiraIssueId);
+ }
+
+ public void reassign(String jiraIssueId, String assignee) {
+ metas.compute(jiraIssueId, (__, jiraIssueMeta) ->
+ new IssueInfo(
+ jiraIssueId,
+ jiraIssueMeta.key(),
+ clock.instant(),
+ Optional.of(assignee),
+ jiraIssueMeta.status()));
+ }
+
+ public void comment(String jiraIssueId, String comment) {
+ touch(jiraIssueId);
+ }
+
+ public void update(String jiraIssueId, String description) {
+ issues.compute(jiraIssueId, (__, issue) ->
+ new Issue(issue.summary(), description, issue.classification().orElse(null)));
+ }
+
+ public String file(Issue issue) {
+ String jiraIssueId = (issues.size() + 1L) + "";
+ Long counter = counters.merge(issue.classification().get().queue(), 0L, (old, __) -> old + 1);
+ String jiraIssueKey = issue.classification().get().queue() + '-' + counter;
+ issues.put(jiraIssueId, issue);
+ metas.put(jiraIssueId, new IssueInfo(jiraIssueId, jiraIssueKey, clock.instant(), null, toDo));
+ return jiraIssueId;
+ }
+
+ public IssueInfo fetch(String jiraIssueId) {
+ return metas.get(jiraIssueId);
+ }
+
+ public List<IssueInfo> fetchSimilarTo(Issue issue) {
+ return issues.entrySet().stream()
+ .filter(entry -> entry.getValue().summary().equals(issue.summary()))
+ .map(Map.Entry::getKey)
+ .map(metas::get)
+ .filter(meta -> meta.status() != done)
+ .collect(Collectors.toList());
+ }
+
+ private void complete(String jiraIssueId) {
+ metas.compute(jiraIssueId, (__, jiraIssueMeta) ->
+ new IssueInfo(
+ jiraIssueId,
+ jiraIssueMeta.key(),
+ clock.instant(),
+ jiraIssueMeta.assignee(),
+ done));
+ }
+
+ private void touch(String jiraIssueId) {
+ metas.compute(jiraIssueId, (__, jiraIssueMeta) ->
+ new IssueInfo(
+ jiraIssueId,
+ jiraIssueMeta.key(),
+ clock.instant(),
+ jiraIssueMeta.assignee(),
+ jiraIssueMeta.status()));
+ }
+
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
new file mode 100644
index 00000000000..cde511a9076
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -0,0 +1,169 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class FailureRedeployerTest {
+
+ @Test
+ public void testRetryingFailedJobsDuringDeployment() {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("canary")
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+ Version version = Version.fromString("5.0");
+ tester.updateVersionStatus(version);
+
+ Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+
+ // New version is released
+ version = Version.fromString("5.1");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+
+ // Test environments pass
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+
+ // Production job fails and is retried
+ tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress
+ tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false);
+ assertEquals("Production job is retried", 1, tester.buildSystem().jobs().size());
+ assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+
+ // Another version is released, which cancels any pending upgrades to lower versions
+ version = Version.fromString("5.2");
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals("Application starts upgrading to new version", 1, tester.buildSystem().jobs().size());
+ assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+
+ // Failure redeployer does not retry failing job for prod.us-east-3 as there's an ongoing deployment
+ tester.clock().advance(Duration.ofMinutes(1));
+ tester.failureRedeployer().maintain();
+ assertFalse("Job is not retried", tester.buildSystem().jobs().stream()
+ .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id())));
+
+ // Test environments pass
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+
+ // Production job fails again and exhausts all immediate retries
+ tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false);
+ tester.buildSystem().takeJobsToRun();
+ tester.clock().advance(Duration.ofMinutes(10));
+ tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, false);
+ assertTrue("Retries exhausted", tester.buildSystem().jobs().isEmpty());
+ assertTrue("Failure is recorded", tester.application(app.id()).deploymentJobs().hasFailures());
+
+ // Failure redeployer retries job
+ tester.clock().advance(Duration.ofMinutes(5));
+ tester.failureRedeployer().maintain();
+ assertEquals("Job is retried", 1, tester.buildSystem().jobs().size());
+
+ // Production job finally succeeds
+ tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+ assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
+ assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures());
+ }
+
+ @Test
+ public void testRetriesDeploymentWithStuckJobs() {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("canary")
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+
+ Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+
+ // staging-test starts, but does not complete
+ assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName());
+ tester.failureRedeployer().maintain();
+ assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty());
+
+ // Just over 12 hours pass, deployment is retried from beginning
+ tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1)));
+ tester.failureRedeployer().maintain();
+ assertEquals(DeploymentJobs.JobType.component.id(), tester.buildSystem().takeJobsToRun().get(0).jobName());
+
+ // Ensure that system-test is trigered after component. Triggering component records a new change, but in this
+ // case there's already a change in progress which we want to discard and start over
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+ assertEquals(DeploymentJobs.JobType.systemTest.id(), tester.buildSystem().jobs().get(0).jobName());
+ }
+
+ @Test
+ public void testRetriesJobsFailingForCurrentChange() {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("canary")
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+ Version version = Version.fromString("5.0");
+ tester.updateVersionStatus(version);
+
+ Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+
+ // New version is released
+ version = Version.fromString("5.1");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+ assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+
+ // system-test fails and exhausts all immediate retries
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, false);
+ tester.buildSystem().takeJobsToRun();
+ tester.clock().advance(Duration.ofMinutes(10));
+ tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, app, false);
+ assertTrue("Retries exhausted", tester.buildSystem().jobs().isEmpty());
+
+ // Another version is released
+ version = Version.fromString("5.2");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+ assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
+
+ // Consume system-test job for 5.2
+ tester.buildSystem().takeJobsToRun();
+
+ // Failure re-deployer does not retry failing system-test job as it failed for an older change
+ tester.clock().advance(Duration.ofMinutes(5));
+ tester.failureRedeployer().maintain();
+ assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java
new file mode 100644
index 00000000000..44d8adf1d15
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobControlTest.java
@@ -0,0 +1,93 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class JobControlTest {
+
+ @Test
+ public void testJobControl() {
+ JobControl jobControl = new JobControl(new MockCuratorDb());
+
+ assertTrue(jobControl.jobs().isEmpty());
+
+ String job1 = "Job1";
+ String job2 = "Job2";
+
+ jobControl.started(job1);
+ jobControl.started(job2);
+ assertEquals(2, jobControl.jobs().size());
+ assertTrue(jobControl.jobs().contains(job1));
+ assertTrue(jobControl.jobs().contains(job2));
+
+ assertTrue(jobControl.isActive(job1));
+ assertTrue(jobControl.isActive(job2));
+
+ jobControl.setActive(job1, false);
+ assertFalse(jobControl.isActive(job1));
+ assertTrue(jobControl.isActive(job2));
+
+ jobControl.setActive(job2, false);
+ assertFalse(jobControl.isActive(job1));
+ assertFalse(jobControl.isActive(job2));
+
+ jobControl.setActive(job1, true);
+ assertTrue(jobControl.isActive(job1));
+ assertFalse(jobControl.isActive(job2));
+
+ jobControl.setActive(job2, true);
+ assertTrue(jobControl.isActive(job1));
+ assertTrue(jobControl.isActive(job2));
+ }
+
+ @Test
+ public void testJobControlMayDeactivateJobs() {
+ JobControl jobControl = new JobControl(new MockCuratorDb());
+
+ ControllerTester tester = new ControllerTester();
+ MockMaintainer mockMaintainer = new MockMaintainer(tester.controller(), jobControl);
+
+ assertTrue(jobControl.jobs().contains("MockMaintainer"));
+
+ assertEquals(0, mockMaintainer.maintenanceInvocations);
+
+ mockMaintainer.run();
+ assertEquals(1, mockMaintainer.maintenanceInvocations);
+
+ jobControl.setActive("MockMaintainer", false);
+ mockMaintainer.run();
+ assertEquals(1, mockMaintainer.maintenanceInvocations);
+
+ jobControl.setActive("MockMaintainer", true);
+ mockMaintainer.run();
+ assertEquals(2, mockMaintainer.maintenanceInvocations);
+ }
+
+ private static class MockMaintainer extends Maintainer {
+
+ int maintenanceInvocations = 0;
+
+ public MockMaintainer(Controller controller, JobControl jobControl) {
+ super(controller, Duration.ofHours(1), jobControl);
+ }
+
+ @Override
+ protected void maintain() {
+ maintenanceInvocations++;
+ }
+
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
new file mode 100644
index 00000000000..a832a591217
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -0,0 +1,132 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.MetricsMock;
+import com.yahoo.vespa.hosted.controller.MetricsMock.MapContext;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
+import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Map;
+
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
+import static org.fest.assertions.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author mortent
+ */
+public class MetricsReporterTest {
+
+ @Test
+ public void test_chef_metrics() throws IOException {
+ ControllerTester tester = new ControllerTester();
+ MetricsMock metricsMock = new MetricsMock();
+ MetricsReporter metricsReporter = setupMetricsReporter(tester.controller(), metricsMock, SystemName.cd);
+ metricsReporter.maintain();
+ assertEquals(2, metricsMock.getMetrics().size());
+
+ Map<MapContext, Map<String, Number>> metrics = metricsMock.getMetricsFilteredByHost("fake-node.test");
+ assertEquals(1, metrics.size());
+ Map.Entry<MapContext, Map<String, Number>> metricEntry = metrics.entrySet().iterator().next();
+ MapContext metricContext = metricEntry.getKey();
+ assertDimension(metricContext, "tenantName", "ciintegrationtests");
+ assertDimension(metricContext, "app", "restart.default");
+ assertDimension(metricContext, "zone", "prod.cd-us-east-1");
+ assertThat(metricEntry.getValue().get(MetricsReporter.convergeMetric).longValue()).isEqualTo(727);
+ }
+
+ @Test
+ public void test_deployment_metrics() throws IOException {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .build();
+ MetricsMock metricsMock = new MetricsMock();
+ MetricsReporter metricsReporter = setupMetricsReporter(tester.controller(), metricsMock, SystemName.cd);
+
+ metricsReporter.maintain();
+ assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric));
+
+ // Deploy 3 apps successfully
+ Application app1 = tester.createApplication("app1", "tenant1", 1, 11L);
+ Application app2 = tester.createApplication("app2", "tenant1", 2, 22L);
+ Application app3 = tester.createApplication("app3", "tenant1", 3, 33L);
+ Application app4 = tester.createApplication("app4", "tenant1", 4, 44L);
+ tester.deployCompletely(app1, applicationPackage);
+ tester.deployCompletely(app2, applicationPackage);
+ tester.deployCompletely(app3, applicationPackage);
+
+ metricsReporter.maintain();
+ assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric));
+
+ // 1 app fails system-test
+ tester.notifyJobCompletion(component, app4, true);
+ tester.deployAndNotify(systemTest, app4, applicationPackage, false);
+
+ metricsReporter.maintain();
+ assertEquals(25.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric));
+ }
+
+ @Test
+ public void it_omits_zone_when_unknown() throws IOException {
+ ControllerTester tester = new ControllerTester();
+ String hostname = "fake-node2.test";
+ MapContext metricContext = getMetricsForHost(tester.controller(), hostname);
+ assertThat(metricContext.getDimensions().get("zone")).isNull();
+ }
+
+ private void assertDimension(MapContext metricContext, String dimensionName, String expectedValue) {
+ assertThat(metricContext.getDimensions().get(dimensionName)).isNotNull().isEqualTo(expectedValue);
+ }
+
+ private MetricsReporter setupMetricsReporter(Controller controller, MetricsMock metricsMock, SystemName system) throws IOException {
+ Chef client = Mockito.mock(Chef.class);
+ PartialNodeResult result = new ObjectMapper()
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+ .readValue(getClass().getClassLoader().getResource("chef_output.json"), PartialNodeResult.class);
+ when(client.partialSearchNodes(anyString(), anyListOf(AttributeMapping.class))).thenReturn(result);
+
+ Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault());
+
+ return new MetricsReporter(controller, metricsMock, client, clock,
+ new JobControl(new MockCuratorDb()), system);
+ }
+
+ private MapContext getMetricsForHost(Controller controller, String hostname) throws IOException {
+ MetricsMock metricsMock = new MetricsMock();
+ MetricsReporter metricsReporter = setupMetricsReporter(controller, metricsMock, SystemName.main);
+ metricsReporter.maintain();
+
+ assertThat(metricsMock.getMetrics()).isNotEmpty();
+
+ Map<MapContext, Map<String, Number>> metrics = metricsMock.getMetricsFilteredByHost(hostname);
+ assertThat(metrics).hasSize(1);
+ Map.Entry<MapContext, Map<String, Number>> metricEntry = metrics.entrySet().iterator().next();
+ return metricEntry.getKey();
+ }
+
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
new file mode 100644
index 00000000000..78b4f7f895f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
@@ -0,0 +1,56 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class OutstandingChangeDeployerTest {
+
+ @Test
+ public void testChangeDeployer() {
+ DeploymentTester tester = new DeploymentTester();
+ OutstandingChangeDeployer deployer = new OutstandingChangeDeployer(tester.controller(), Duration.ofMinutes(10),
+ new JobControl(new MockCuratorDb()));
+
+ tester.createAndDeploy("app1", 11, "default");
+ tester.createAndDeploy("app2", 22, "default");
+
+ Version version = new Version(5, 2);
+ tester.deploymentTrigger().triggerChange(tester.application("app1").id(), new Change.VersionChange(version));
+
+ assertEquals(new Change.VersionChange(version), tester.application("app1").deploying().get());
+ assertFalse(tester.application("app1").hasOutstandingChange());
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, tester.application("app1"), true);
+ assertTrue(tester.application("app1").hasOutstandingChange());
+ assertEquals(1, tester.buildSystem().jobs().size());
+
+ deployer.maintain();
+ assertEquals("No effect as job is in progress", 1, tester.buildSystem().jobs().size());
+
+ tester.deployCompletely("app1");
+ assertEquals("Upgrade done", 0, tester.buildSystem().jobs().size());
+
+ deployer.maintain();
+ List<BuildService.BuildJob> jobs = tester.buildSystem().jobs();
+ assertEquals(1, jobs.size());
+ assertEquals(11, jobs.get(0).projectId());
+ assertEquals(DeploymentJobs.JobType.systemTest.id(), jobs.get(0).jobName());
+ assertFalse(tester.application("app1").hasOutstandingChange());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
new file mode 100644
index 00000000000..e5afcec87ad
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -0,0 +1,304 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class UpgraderTest {
+
+ @Test
+ public void testUpgrading() {
+ // --- Setup
+ DeploymentTester tester = new DeploymentTester();
+ tester.upgrader().maintain();
+ assertEquals("No system version: Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ Version version = Version.fromString("5.0"); // (lower than the hardcoded version in the config server client)
+ tester.updateVersionStatus(version);
+
+ tester.upgrader().maintain();
+ assertEquals("No applications: Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ // Setup applications
+ Application canary0 = tester.createAndDeploy("canary0", 0, "canary");
+ Application canary1 = tester.createAndDeploy("canary1", 1, "canary");
+ Application default0 = tester.createAndDeploy("default0", 2, "default");
+ Application default1 = tester.createAndDeploy("default1", 3, "default");
+ Application default2 = tester.createAndDeploy("default2", 4, "default");
+ Application conservative0 = tester.createAndDeploy("conservative0", 5, "conservative");
+
+ tester.upgrader().maintain();
+ assertEquals("All already on the right version: Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ // --- A new version is released - everything goes smoothly
+ version = Version.fromString("5.1");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+
+ assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size());
+ tester.completeUpgrade(canary0, version, "canary");
+ assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get());
+
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals("One canary pending; nothing else", 1, tester.buildSystem().jobs().size());
+
+ tester.completeUpgrade(canary1, version, "canary");
+
+ tester.updateVersionStatus(version);
+ assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+ assertEquals("Canaries done: Should upgrade defaults", 3, tester.buildSystem().jobs().size());
+
+ tester.completeUpgrade(default0, version, "default");
+ tester.completeUpgrade(default1, version, "default");
+ tester.completeUpgrade(default2, version, "default");
+
+ tester.updateVersionStatus(version);
+ assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+ assertEquals("Normals done: Should upgrade conservatives", 1, tester.buildSystem().jobs().size());
+ tester.completeUpgrade(conservative0, version, "conservative");
+
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ // --- A new version is released - which fails a Canary
+ version = Version.fromString("5.2");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+
+ assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size());
+ tester.completeUpgradeWithError(canary0, version, "canary", DeploymentJobs.JobType.stagingTest);
+ assertEquals("Other Canary was cancelled", 2, tester.buildSystem().jobs().size());
+
+ tester.updateVersionStatus(version);
+ assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+ assertEquals("Version broken, but Canaries should keep trying", 2, tester.buildSystem().jobs().size());
+
+ // --- A new version is released - which repairs the Canary app and fails a default
+ version = Version.fromString("5.3");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+
+ assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size());
+ tester.completeUpgrade(canary0, version, "canary");
+ assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get());
+
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals("One canary pending; nothing else", 1, tester.buildSystem().jobs().size());
+
+ tester.completeUpgrade(canary1, version, "canary");
+
+ tester.updateVersionStatus(version);
+ assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+
+ assertEquals("Canaries done: Should upgrade defaults", 3, tester.buildSystem().jobs().size());
+
+ tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.stagingTest);
+ tester.completeUpgrade(default1, version, "default");
+ tester.completeUpgrade(default2, version, "default");
+
+ tester.updateVersionStatus(version);
+ assertEquals("Not enough evidence to mark this neither broken nor high",
+ VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+ assertEquals("Upgrade with error should retry", 1, tester.buildSystem().jobs().size());
+
+ // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold
+ // Deploy application change
+ tester.deployCompletely("default0");
+ // Complete upgrade
+ tester.upgrader().maintain();
+ tester.completeUpgrade(default0, version, "default");
+
+ tester.updateVersionStatus(version);
+ assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+ assertEquals("Normals done: Should upgrade conservatives", 1, tester.buildSystem().jobs().size());
+ tester.completeUpgrade(conservative0, version, "conservative");
+
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size());
+ }
+
+ @Test
+ public void testUpgradingToVersionWhichBreaksSomeNonCanaries() {
+ // --- Setup
+ DeploymentTester tester = new DeploymentTester();
+ tester.upgrader().maintain();
+ assertEquals("No system version: Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ Version version = Version.fromString("5.0"); // (lower than the hardcoded version in the config server client)
+ tester.updateVersionStatus(version);
+
+ tester.upgrader().maintain();
+ assertEquals("No applications: Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ // Setup applications
+ Application canary0 = tester.createAndDeploy("canary0", 0, "canary");
+ Application canary1 = tester.createAndDeploy("canary1", 1, "canary");
+ Application default0 = tester.createAndDeploy("default0", 2, "default");
+ Application default1 = tester.createAndDeploy("default1", 3, "default");
+ Application default2 = tester.createAndDeploy("default2", 4, "default");
+ Application default3 = tester.createAndDeploy("default3", 5, "default");
+ Application default4 = tester.createAndDeploy("default4", 6, "default");
+ Application default5 = tester.createAndDeploy("default5", 7, "default");
+ Application default6 = tester.createAndDeploy("default6", 8, "default");
+ Application default7 = tester.createAndDeploy("default7", 9, "default");
+ Application default8 = tester.createAndDeploy("default8", 10, "default");
+ Application default9 = tester.createAndDeploy("default9", 11, "default");
+
+ tester.upgrader().maintain();
+ assertEquals("All already on the right version: Nothing to do", 0, tester.buildSystem().jobs().size());
+
+ // --- A new version is released
+ version = Version.fromString("5.1");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+
+ assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size());
+ tester.completeUpgrade(canary0, version, "canary");
+ assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get());
+
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals("One canary pending; nothing else", 1, tester.buildSystem().jobs().size());
+
+ tester.completeUpgrade(canary1, version, "canary");
+
+ tester.updateVersionStatus(version);
+ assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
+ tester.upgrader().maintain();
+ assertEquals("Canaries done: Should upgrade defaults", 10, tester.buildSystem().jobs().size());
+
+ tester.completeUpgrade(default0, version, "default");
+ tester.completeUpgradeWithError(default1, version, "default", DeploymentJobs.JobType.systemTest);
+ tester.completeUpgradeWithError(default2, version, "default", DeploymentJobs.JobType.systemTest);
+ tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.systemTest);
+ tester.completeUpgradeWithError(default4, version, "default", DeploymentJobs.JobType.systemTest);
+
+ // > 40% and at least 4 failed - version is broken
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
+ assertEquals("Upgrades are cancelled", 0, tester.buildSystem().jobs().size());
+ }
+
+ @Test
+ public void testDeploymentAlreadyInProgressForUpgrade() {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("canary")
+ .environment(Environment.prod)
+ .region("us-east-3")
+ .build();
+ Version version = Version.fromString("5.0");
+ tester.updateVersionStatus(version);
+
+ Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+
+ tester.upgrader().maintain();
+ assertEquals("Application is on expected version: Nothing to do", 0,
+ tester.buildSystem().jobs().size());
+
+ // New version is released
+ version = Version.fromString("5.1");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+ tester.upgrader().maintain();
+
+ // system-test completes successfully
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+
+ // staging-test fails multiple times, exhausts retries and failure is recorded
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, false);
+ tester.buildSystem().takeJobsToRun();
+ tester.clock().advance(Duration.ofMinutes(10));
+ tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, app, false);
+ assertTrue("Retries exhausted", tester.buildSystem().jobs().isEmpty());
+ assertTrue("Failure is recorded", tester.application(app.id()).deploymentJobs().hasFailures());
+ assertTrue("Application has pending change", tester.application(app.id()).deploying().isPresent());
+
+ // New version is released
+ version = Version.fromString("5.2");
+ tester.updateVersionStatus(version);
+ assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+
+ // Upgrade is scheduled. system-tests starts, but does not complete
+ tester.upgrader().maintain();
+ assertTrue("Application still has failures", tester.application(app.id()).deploymentJobs().hasFailures());
+ assertEquals(1, tester.buildSystem().jobs().size());
+ tester.buildSystem().takeJobsToRun();
+
+ // Upgrader runs again, nothing happens as there's already a job in progress for this change
+ tester.upgrader().maintain();
+ assertTrue("No more jobs triggered at this time", tester.buildSystem().jobs().isEmpty());
+ }
+
+ // TODO: Remove when corp-prod special casing is no longer needed
+ @Test
+ public void upgradesCanariesToControllerVersion() {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("canary")
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+
+ Version version = Version.fromString("5.0"); // Lower version than controller (6.10)
+ tester.updateVersionStatus(version);
+
+ // Application is on 5.0
+ Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+ tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true);
+
+ // Canary in prod.corp-us-east-1 is upgraded to controller version
+ tester.upgrader().maintain();
+ assertEquals("Upgrade started", 1, tester.buildSystem().jobs().size());
+ assertEquals(Vtag.currentVersion, ((Change.VersionChange) tester.application(app.id()).deploying().get()).version());
+ tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+ tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true);
+
+ // System is upgraded to newer version, no upgrade triggered for canary as version is lower than controller
+ version = Version.fromString("5.1");
+ tester.updateVersionStatus(version);
+ tester.upgrader().maintain();
+ assertTrue("No more jobs triggered", tester.buildSystem().jobs().isEmpty());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java
new file mode 100644
index 00000000000..a7cecda3695
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdaterTest.java
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Collections;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class VersionStatusUpdaterTest {
+
+ /** Test that this job updates the status. Test of the content of the update is in VersionStatusTest */
+ @Test
+ public void testVersionUpdating() {
+ ControllerTester tester = new ControllerTester();
+ tester.controller().updateVersionStatus(new VersionStatus(Collections.emptyList()));
+ assertFalse(tester.controller().versionStatus().systemVersion().isPresent());
+
+ VersionStatusUpdater updater = new VersionStatusUpdater(tester.controller(), Duration.ofMinutes(3),
+ new JobControl(new MockCuratorDb()));
+ updater.maintain();
+ assertTrue(tester.controller().versionStatus().systemVersion().isPresent());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
new file mode 100644
index 00000000000..645e38d0f2d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -0,0 +1,196 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
+import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author bratseth
+ */
+public class ApplicationSerializerTest {
+
+ private static final ApplicationSerializer applicationSerializer = new ApplicationSerializer();
+
+ private static final Zone zone1 = new Zone(Environment.from("prod"), RegionName.from("us-west-1"));
+ private static final Zone zone2 = new Zone(Environment.from("prod"), RegionName.from("us-east-3"));
+
+ @Test
+ public void testSerialization() {
+ ControllerTester tester = new ControllerTester();
+ DeploymentSpec deploymentSpec = DeploymentSpec.fromXml("<deployment version='1.0'>" +
+ " <staging/>" +
+ "</deployment>");
+ ValidationOverrides validationOverrides = ValidationOverrides.fromXml("<validation-overrides version='1.0'>" +
+ " <allow until='2017-06-15'>deployment-removal</allow>" +
+ "</validation-overrides>");
+
+ List<Deployment> deployments = new ArrayList<>();
+ ApplicationRevision revision1 = ApplicationRevision.from("appHash1");
+ ApplicationRevision revision2 = ApplicationRevision.from("appHash2", new SourceRevision("repo1", "branch1", "commit1"));
+ deployments.add(new Deployment(zone1, revision1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3)));
+ deployments.add(new Deployment(zone2, revision2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5)));
+
+ Optional<Long> projectId = Optional.of(123L);
+ List<JobStatus> statusList = new ArrayList<>();
+
+ statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest)
+ .withTriggering(Version.fromString("5.6.7"), Optional.empty(), Instant.ofEpochMilli(7))
+ .withCompletion(Optional.empty(), Instant.ofEpochMilli(8), tester.controller()));
+ statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest)
+ .withTriggering(Version.fromString("5.6.6"), Optional.empty(), Instant.ofEpochMilli(5))
+ .withCompletion(Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller()));
+
+ DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty(), false);
+
+ Application original = new Application(ApplicationId.from("t1", "a1", "i1"),
+ deploymentSpec,
+ validationOverrides,
+ deployments, deploymentJobs,
+ Optional.of(new Change.VersionChange(Version.fromString("6.7"))),
+ true);
+
+ Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original));
+
+ assertEquals(original.id(), serialized.id());
+
+ assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
+ assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
+
+ assertEquals(2, serialized.deployments().size());
+ assertEquals(original.deployments().get(zone1).revision(), serialized.deployments().get(zone1).revision());
+ assertEquals(original.deployments().get(zone2).revision(), serialized.deployments().get(zone2).revision());
+ assertEquals(original.deployments().get(zone1).version(), serialized.deployments().get(zone1).version());
+ assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version());
+ assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at());
+ assertEquals(original.deployments().get(zone2).at(), serialized.deployments().get(zone2).at());
+
+ assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId());
+ assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size());
+ assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest),
+ serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest));
+ assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest),
+ serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest));
+ assertEquals(original.deploymentJobs().failingSince(), serialized.deploymentJobs().failingSince());
+ assertEquals(original.deploymentJobs().isSelfTriggering(), serialized.deploymentJobs().isSelfTriggering());
+
+ assertEquals(original.hasOutstandingChange(), serialized.hasOutstandingChange());
+
+ assertEquals(original.deploying(), serialized.deploying());
+
+ { // test more deployment serialization cases
+ Application original2 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1"))));
+ Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2));
+ assertEquals(original2.deploying(), serialized2.deploying());
+ assertEquals(((Change.ApplicationChange)serialized2.deploying().get()).revision().get().source(),
+ ((Change.ApplicationChange)original2.deploying().get()).revision().get().source());
+
+ Application original3 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1",
+ new SourceRevision("a", "b", "c")))));
+ Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3));
+ assertEquals(original3.deploying(), serialized2.deploying());
+ assertEquals(((Change.ApplicationChange)serialized3.deploying().get()).revision().get().source(),
+ ((Change.ApplicationChange)original3.deploying().get()).revision().get().source());
+
+ Application original4 = original.withDeploying(Optional.empty());
+ Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4));
+ assertEquals(original4.deploying(), serialized4.deploying());
+ }
+ }
+
+ @Test
+ public void testLegacySerialization() throws IOException {
+ Application applicationWithSuccessfulJob = applicationSerializer.fromSlime(applicationSlime(false));
+ assertFalse("No job error for successful job", applicationWithSuccessfulJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().isPresent());
+
+ Application applicationWithFailingJob = applicationSerializer.fromSlime(applicationSlime(true));
+ assertEquals(JobError.unknown, applicationWithFailingJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().get());
+ }
+
+ // TODO: Remove after Aug 2017
+ @Test
+ public void serializeWithRemovedZone() throws Exception {
+ String json = "{\n" +
+ " \"id\": \"t1:a1:i1\",\n" +
+ " \"deploymentSpecField\": \"<deployment version='1.0'/>\",\n" +
+ " \"deploymentJobs\": {\n" +
+ " \"projectId\": 123,\n" +
+ " \"jobStatus\": [\n" +
+ " {\n" +
+ " \"jobType\": \"system-test\",\n" +
+ " \"version\": \"5.6.7\",\n" +
+ " \"completionTime\": 7,\n" +
+ " \"lastTriggered\": 8\n" +
+ " },\n" +
+ " {\n" +
+ " \"jobType\": \"production-ap-aue-1\",\n" +
+ " \"version\": \"5.6.7\",\n" +
+ " \"completionTime\": 7,\n" +
+ " \"lastTriggered\": 8\n" +
+ " },\n" +
+ " {\n" +
+ " \"jobType\": \"staging-test\",\n" +
+ " \"version\": \"5.6.7\",\n" +
+ " \"completionTime\": 7,\n" +
+ " \"lastTriggered\": 8\n" +
+ " }\n" +
+ " ],\n" +
+ " \"selfTriggering\": false\n" +
+ " }\n" +
+ "}\n";
+ Application app = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(json.getBytes(StandardCharsets.UTF_8)));
+ assertEquals(2, app.deploymentJobs().jobStatus().size());
+ }
+
+ private Slime applicationSlime(boolean error) {
+ return SlimeUtils.jsonToSlime(applicationJson(error).getBytes(StandardCharsets.UTF_8));
+ }
+
+ private String applicationJson(boolean error) {
+ return
+ "{\n" +
+ " \"id\": \"t1:a1:i1\",\n" +
+ " \"deploymentSpecField\": \"<deployment version='1.0'/>\",\n" +
+ " \"deploymentJobs\": {\n" +
+ " \"projectId\": 123,\n" +
+ " \"jobStatus\": [\n" +
+ " {\n" +
+ " \"jobType\": \"system-test\",\n" +
+ " \"version\": \"5.6.7\",\n" +
+ " \"completionTime\": 7,\n" +
+ (error ? " \"jobError\": \"" + JobError.unknown + "\",\n" : "") +
+ " \"lastTriggered\": 8\n" +
+ " }\n" +
+ " ],\n" +
+ " \"selfTriggering\": false\n" +
+ " }\n" +
+ "}\n";
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
new file mode 100644
index 00000000000..348b9c92614
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
@@ -0,0 +1,101 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.application.container.JDisc;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.TestIdentities;
+import com.yahoo.vespa.hosted.controller.api.Tenant;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
+import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Optional;
+
+/**
+ * Provides testing of controller functionality accessed through the container
+ *
+ * @author bratseth
+ */
+public class ContainerControllerTester {
+
+ private final ContainerTester containerTester;
+ private final Controller controller;
+
+ public ContainerControllerTester(JDisc container, String responseFilePath) {
+ containerTester = new ContainerTester(container, responseFilePath);
+ controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller");
+ }
+
+ public Controller controller() { return controller; }
+
+ /** Returns the wrapped generic container tester */
+ public ContainerTester containerTester() { return containerTester; }
+
+ public Application createApplication() {
+ AthensDomain domain1 = addTenantAthensDomain("domain1", "mytenant");
+ controller.tenants().addTenant(Tenant.createAthensTenant(new TenantId("tenant1"), domain1,
+ new Property("property1"),
+ Optional.of(new PropertyId("1234"))),
+ Optional.of(TestIdentities.userNToken));
+ ApplicationId app = ApplicationId.from("tenant1", "application1", "default");
+ return controller.applications().createApplication(app, Optional.of(TestIdentities.userNToken));
+ }
+
+ public Application deploy(Application application, ApplicationPackage applicationPackage, Zone zone, long projectId) {
+ ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(projectId));
+ GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1"));
+ controller.applications().deployApplication(application.id(),
+ zone,
+ applicationPackage,
+ new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), Optional.empty(), false, false));
+ return application;
+ }
+
+ public void notifyJobCompletion(ApplicationId applicationId, long projectId, boolean success, DeploymentJobs.JobType job) {
+ controller().applications().notifyJobCompletion(new DeploymentJobs.JobReport(applicationId, job, projectId, 1L,
+ success ? Optional.empty() : Optional.of(DeploymentJobs.JobError.unknown),
+ false, false));
+ }
+
+ public AthensDomain addTenantAthensDomain(String domainName, String userName) {
+ Athens athens = (AthensMock) containerTester.container().components().getComponent(
+ "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock"
+ );
+ ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory();
+ AthensDomain athensDomain = new AthensDomain(domainName);
+ AthensDbMock.Domain domain = new AthensDbMock.Domain(athensDomain);
+ domain.markAsVespaTenant();
+ domain.admin(new AthensPrincipal(new AthensDomain("domain"), new UserId(userName)));
+ mock.getSetup().addDomain(domain);
+ return athensDomain;
+ }
+
+ // ---- Delegators:
+
+ public void assertResponse(Request request, File expectedResponse) throws IOException {
+ containerTester.assertResponse(request, expectedResponse);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
new file mode 100644
index 00000000000..7a9e74a3c27
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
@@ -0,0 +1,136 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.application.container.JDisc;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.application.container.handler.Response;
+import com.yahoo.collections.Pair;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.Type;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Provides testing of JSON container responses
+ *
+ * @author bratseth
+ */
+public class ContainerTester {
+
+ private final JDisc container;
+ private final String responseFilePath;
+
+ public ContainerTester(JDisc container, String responseFilePath) {
+ this.container = container;
+ this.responseFilePath = responseFilePath;
+ }
+
+ public JDisc container() { return container; }
+
+ public void updateSystemVersion() {
+ Controller controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller");
+ controller.updateVersionStatus(VersionStatus.compute(controller));
+ }
+
+ public void assertResponse(Request request, File responseFile) throws IOException {
+ assertResponse(request, responseFile, 200);
+ }
+
+ public void assertResponse(Request request, File responseFile, int expectedStatusCode) throws IOException {
+ String expectedResponse = IOUtils.readFile(new File(responseFilePath + responseFile.toString()));
+ expectedResponse = include(expectedResponse);
+ Response response = container.handleRequest(request);
+ Slime expectedSlime = SlimeUtils.jsonToSlime(expectedResponse.getBytes(StandardCharsets.UTF_8));
+ Set<String> fieldsToCensor = fieldsToCensor(null, expectedSlime.get(), new HashSet<>());
+ Slime responseSlime = SlimeUtils.jsonToSlime(response.getBody());
+ List<Pair<String,String>> replaceStrings = new ArrayList<>();
+ buildReplaceStrings(null, responseSlime.get(), fieldsToCensor, replaceStrings);
+
+ String body = response.getBodyAsString();
+ assertEquals("Status code. Response body was: " + body, expectedStatusCode, response.getStatus());
+ assertEquals(responseFile.toString(), new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8),
+ replace(new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8), replaceStrings));
+ }
+
+ public void assertResponse(Request request, String expectedResponse) throws IOException {
+ assertResponse(request, expectedResponse, 200);
+ }
+
+ public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException {
+ Response response = container.handleRequest(request);
+ assertEquals("Status code", expectedStatusCode, response.getStatus());
+ assertEquals(expectedResponse, response.getBodyAsString());
+ }
+
+ private Set<String> fieldsToCensor(String fieldNameOrNull, Inspector value, Set<String> fieldsToCensor) {
+ switch (value.type()) {
+ case ARRAY: value.traverse((ArrayTraverser)(int index, Inspector element) -> fieldsToCensor(null, element, fieldsToCensor)); break;
+ case OBJECT: value.traverse((String fieldName, Inspector fieldValue) -> fieldsToCensor(fieldName, fieldValue, fieldsToCensor)); break;
+ case STRING: if (fieldNameOrNull != null && "(ignore)".equals(value.asString())) fieldsToCensor.add(fieldNameOrNull); break;
+ }
+ return fieldsToCensor;
+ }
+
+ private void buildReplaceStrings(String fieldNameOrNull, Inspector value, Set<String> fieldsToCensor,
+ List<Pair<String,String>> replaceStrings) {
+ switch (value.type()) {
+ case ARRAY: value.traverse((ArrayTraverser)(int index, Inspector element) -> buildReplaceStrings(null, element, fieldsToCensor, replaceStrings)); break;
+ case OBJECT: value.traverse((String fieldName, Inspector fieldValue) -> buildReplaceStrings(fieldName, fieldValue, fieldsToCensor, replaceStrings)); break;
+ default: replaceString(fieldNameOrNull, value, fieldsToCensor, replaceStrings);
+ }
+ }
+
+ private void replaceString(String fieldName, Inspector fieldValue,
+ Set<String> fieldsToCensor, List<Pair<String,String>> replaceStrings) {
+ if (fieldName == null) return;
+ if ( ! fieldsToCensor.contains(fieldName)) return;
+
+ String fromString;
+ if ( fieldValue.type().equals(Type.STRING))
+ fromString = "\"" + fieldName + "\":\"" + fieldValue.asString() + "\"";
+ else if ( fieldValue.type().equals(Type.LONG))
+ fromString = "\"" + fieldName + "\":" + fieldValue.asLong();
+ else
+ throw new IllegalArgumentException("Can only censor strings and longs");
+ String toString = "\"" + fieldName + "\":\"(ignore)\"";
+ replaceStrings.add(new Pair<>(fromString, toString));
+ }
+
+ private String replace(String json, List<Pair<String,String>> replaceStrings) {
+ for (Pair<String,String> replaceString : replaceStrings)
+ json = json.replace(replaceString.getFirst(), replaceString.getSecond());
+ return json;
+ }
+
+ /** Replaces @include(localFile) with the content of the file */
+ private String include(String response) throws IOException {
+ // Please don't look at this code
+ int includeIndex = response.indexOf("@include(");
+ if (includeIndex < 0) return response;
+ String prefix = response.substring(0, includeIndex);
+ String rest = response.substring(includeIndex + "@include(".length());
+ int filenameEnd = rest.indexOf(")");
+ String includeFileName = rest.substring(0, filenameEnd);
+ String includedContent = IOUtils.readFile(new File(responseFilePath + includeFileName));
+ includedContent = include(includedContent);
+ String postFix = rest.substring(filenameEnd + 1);
+ postFix = include(postFix);
+ return prefix + includedContent + postFix;
+ }
+
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
new file mode 100644
index 00000000000..8b2595c6254
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -0,0 +1,85 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.application.Networking;
+import com.yahoo.application.container.JDisc;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.application.container.handler.Response;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Superclass of REST API tests which needs to set up a functional container instance.
+ *
+ * This is a test superclass, not a tester because we need the start and stop methods.
+ *
+ * DO NOT ADD ANYTHING HERE: If you need additional fields and methods, create a tester
+ * which gets the container instance at construction time (in the test method) instead.
+ *
+ * @author bratseth
+ */
+public class ControllerContainerTest {
+
+ protected JDisc container;
+ @Before
+ public void startContainer() { container = JDisc.fromServicesXml(controllerServicesXml, Networking.disable); }
+ @After
+ public void stopContainer() { container.close(); }
+
+ private final String controllerServicesXml =
+ "<jdisc version='1.0'>" +
+ " <config name='vespa.hosted.zone.config.zone'>" +
+ " <system>main</system>" +
+ " </config>" +
+ " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.jira.JiraMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.ContactsMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingIssues'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.PropertiesMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.ConfigServerClientMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.ZoneRegistryMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.Controller'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.cost.MockInsightBackend'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.cost.CostMock'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.integration.MockMetricsService'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.restapi.application.MockAuthorizer'/>" +
+ " <component id='com.yahoo.vespa.hosted.controller.routing.MockRoutingGenerator'/>" +
+ " <component id='com.yahoo.vespa.hosted.rotation.MemoryRotationRepository'/>" +
+ " <handler id='com.yahoo.vespa.hosted.controller.restapi.RootHandler'>" +
+ " <binding>http://*/</binding>" +
+ " </handler>" +
+ " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>" +
+ " <binding>http://*/application/v4/*</binding>" +
+ " </handler>" +
+ " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>" +
+ " <binding>http://*/deployment/v1/*</binding>" +
+ " </handler>" +
+ " <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>" +
+ " <binding>http://*/controller/v1/*</binding>" +
+ " </handler>" +
+ " <handler id='com.yahoo.vespa.hosted.controller.restapi.screwdriver.ScrewdriverApiHandler'>" +
+ " <binding>http://*/screwdriver/v1/*</binding>" +
+ " </handler>" +
+ "</jdisc>";
+
+ protected void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException {
+ Response response = container.handleRequest(request);
+ // Compare both status and message at once for easier diagnosis
+ assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage,
+ "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString());
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java
new file mode 100644
index 00000000000..9534dd65d7f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/RootHandlerTest.java
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi;
+
+import com.yahoo.application.container.handler.Request;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * @author bratseth
+ */
+public class RootHandlerTest extends ControllerContainerTest {
+
+ @Test
+ public void testRootRequest() throws IOException {
+ ContainerTester tester = new ContainerTester(container,
+ "src/test/java/com/yahoo/vespa/hosted/controller/restapi/");
+ tester.assertResponse(new Request("http://localhost:8080/"),
+ new File("root-response.json"), 200);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
new file mode 100644
index 00000000000..97e1bac35c8
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -0,0 +1,598 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.ConfigServerClientMock;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost;
+import com.yahoo.vespa.hosted.controller.api.integration.cost.ClusterCost;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.cost.MockInsightBackend;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author bratseth
+ */
+public class ApplicationApiTest extends ControllerContainerTest {
+
+ private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/";
+ private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+ private static final String athensUserDomain = "domain1";
+ private static final String athensScrewdriverDomain = "screwdriver-domain";
+
+ @Test
+ public void testApplicationApi() throws IOException {
+ ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles);
+ ContainerTester tester = controllerTester.containerTester();
+ tester.updateSystemVersion();
+
+ addTenantAthensDomain(athensUserDomain, "mytenant"); // (Necessary but not provided in this API)
+
+ // GET API root
+ tester.assertResponse(request("/application/v4/", "", Request.Method.GET),
+ new File("root.json"));
+ // GET athens domains
+ tester.assertResponse(request("/application/v4/athensDomain/", "", Request.Method.GET),
+ new File("athensDomain-list.json"));
+ // GET OpsDB properties
+ tester.assertResponse(request("/application/v4/property/", "", Request.Method.GET),
+ new File("property-list.json"));
+ // GET cookie freshness
+ tester.assertResponse(request("/application/v4/cookiefreshness/", "", Request.Method.GET),
+ new File("cookiefreshness.json"));
+ // POST (add) a tenant without property ID
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST),
+ new File("tenant-without-applications.json"));
+ // PUT (modify) a tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.PUT),
+ new File("tenant-without-applications.json"));
+ // GET the authenticated user (with associated tenants)
+ tester.assertResponse(request("/application/v4/user", "", Request.Method.GET),
+ new File("user.json"));
+ // GET all tenants
+ tester.assertResponse(request("/application/v4/tenant/", "", Request.Method.GET),
+ new File("tenant-list.json"));
+ // POST (create) an application
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.POST),
+ new File("application-reference.json"));
+ // GET a tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.GET),
+ new File("tenant-with-application.json"));
+ // GET tenant applications
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/", "", Request.Method.GET),
+ new File("application-list.json"));
+ // POST triggering of a full deployment to an application (if version is omitted, current system version is used)
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST),
+ new File("application-deployment.json"));
+
+ // POST (deploy) an application to a zone - manual user deployment
+ HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty());
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy",
+ entity,
+ Request.Method.POST,
+ athensUserDomain, "mytenant"),
+ new File("deploy-result.json"));
+
+ // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline.
+ ApplicationId id = ApplicationId.from("tenant1", "application1", "default");
+ long screwdriverProjectId = 123;
+
+ addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // (Necessary but not provided in this API)
+
+ // ... systemtest
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/test-region/instance/default/",
+ createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)),
+ Request.Method.POST,
+ athensScrewdriverDomain, "screwdriveruser1"),
+ new File("deploy-result.json"));
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/test-region/instance/default",
+ "",
+ Request.Method.DELETE),
+ "Deactivated tenant/tenant1/application/application1/environment/test/region/test-region/instance/default");
+ controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.systemTest); // Called through the separate screwdriver/v1 API
+
+ // ... staging
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/staging-region/instance/default/",
+ createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)),
+ Request.Method.POST,
+ athensScrewdriverDomain, "screwdriveruser1"),
+ new File("deploy-result.json"));
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/staging-region/instance/default",
+ "",
+ Request.Method.DELETE),
+ "Deactivated tenant/tenant1/application/application1/environment/staging/region/staging-region/instance/default");
+ controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.stagingTest);
+
+ // ... prod zone
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/",
+ createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)),
+ Request.Method.POST,
+ athensScrewdriverDomain, "screwdriveruser1"),
+ new File("deploy-result.json"));
+ controllerTester.notifyJobCompletion(id, screwdriverProjectId, false, DeploymentJobs.JobType.productionCorpUsEast1);
+
+ // GET tenant screwdriver projects
+ tester.assertResponse(request("/application/v4/tenant-pipeline/", "", Request.Method.GET),
+ new File("tenant-pipelines.json"));
+ // GET tenant application deployments
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET),
+ new File("application.json"));
+ // GET an application deployment
+ addMockObservedApplicationCost("tenant1", "application1", "default");
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET),
+ new File("deployment.json"));
+ // POST a 'restart application' command
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart",
+ "",
+ Request.Method.POST),
+ "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default");
+ // POST a 'restart application' command with a host filter (other filters not supported yet)
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1",
+ "",
+ Request.Method.POST),
+ "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default");
+ // POST a 'log' command
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/log",
+ "",
+ Request.Method.POST),
+ new File("log-response.json")); // Proxied to config server, not sure about the expected return format
+ // GET (wait for) convergence
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", "", Request.Method.GET),
+ new File("convergence.json"));
+ // GET services
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", "", Request.Method.GET),
+ new File("services.json"));
+ // GET service
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "", Request.Method.GET),
+ new File("service.json"));
+ // DELETE (deactivate) a deployment - dev
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default",
+ "",
+ Request.Method.DELETE),
+ "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default");
+ // DELETE (deactivate) a deployment - prod
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default",
+ "",
+ Request.Method.DELETE),
+ "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default");
+ // DELETE an application
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.DELETE),
+ "");
+ // DELETE a tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE),
+ new File("tenant-without-applications.json"));
+
+ // PUT (create) the authenticated user
+ tester.assertResponse(request("/application/v4/user?user=newuser&domain=by",
+ new byte[0],
+ Request.Method.PUT,
+ athensUserDomain, "newuser", "application/json"),
+ new File("create-user-response.json"));
+ // OPTIONS return 200 OK
+ tester.assertResponse(request("/application/v4/", "", Request.Method.OPTIONS),
+ "");
+
+ // Add another Athens domain, so we can try to create more tenants
+ addTenantAthensDomain("domain2", "mytenant"); // New domain to test tenant w/property ID
+ // POST (add) a tenant with property ID
+ tester.assertResponse(request("/application/v4/tenant/tenant2",
+ "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}",
+ Request.Method.POST),
+ new File("tenant-without-applications-with-id.json"));
+ // PUT (modify) a tenant with property ID
+ tester.assertResponse(request("/application/v4/tenant/tenant2",
+ "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}",
+ Request.Method.PUT),
+ new File("tenant-without-applications-with-id.json"));
+ // GET a tenant with property ID
+ tester.assertResponse(request("/application/v4/tenant/tenant2", "", Request.Method.GET),
+ new File("tenant-without-applications-with-id.json"));
+
+ // Test legacy OpsDB tenants
+ // POST (add) an OpsDB tenant with property ID
+ tester.assertResponse(request("/application/v4/tenant/tenant3",
+ "{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}",
+ Request.Method.POST),
+ new File("opsdb-tenant-with-id-without-applications.json"));
+ // PUT (modify) the OpsDB tenant to set another property
+ tester.assertResponse(request("/application/v4/tenant/tenant3",
+ "{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}",
+ Request.Method.PUT),
+ new File("opsdb-tenant-with-new-id-without-applications.json"));
+
+ // GET global rotation status
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", "", Request.Method.GET),
+ new File("global-rotation.json"));
+
+ // GET global rotation override status
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", "", Request.Method.GET),
+ new File("global-rotation-get.json"));
+
+ // SET global rotation override status
+ tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.PUT),
+ new File("global-rotation-put.json"));
+
+ // DELETE global rotation override status
+ tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.DELETE),
+ new File("global-rotation-delete.json"));
+
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", "", Request.Method.POST),
+ "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}");
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST),
+ "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}");
+ }
+
+ @Test
+ public void testErrorResponses() throws IOException, URISyntaxException {
+ ContainerTester tester = new ContainerTester(container, responseFiles);
+ tester.updateSystemVersion();
+ addTenantAthensDomain("domain1", "mytenant");
+
+ // PUT (update) non-existing tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.PUT),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}",
+ 404);
+
+ // GET non-existing tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "",
+ Request.Method.GET),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}",
+ 404);
+
+ // GET non-existing application
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.GET),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}",
+ 404);
+
+ // GET non-existing deployment
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east/instance/default",
+ "",
+ Request.Method.GET),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}",
+ 404);
+
+ // POST (add) a tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST),
+ new File("tenant-without-applications.json"));
+
+ // POST (add) another tenant under the same domain
+ tester.assertResponse(request("/application/v4/tenant/tenant2",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST),
+ "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create tenant 'tenant2': The Athens domain 'domain1' is already connected to tenant 'tenant1'\"}",
+ 400);
+
+ // Add the same tenant again
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST),
+ "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}",
+ 400);
+
+ // POST (create) an (empty) application
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.POST),
+ new File("application-reference.json"));
+
+ // Create the same application again
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.POST),
+ "{\"error-code\":\"BAD_REQUEST\",\"message\":\"An application with id 'tenant1.application1' already exists\"}",
+ 400);
+
+ ConfigServerClientMock configServer = (ConfigServerClientMock)container.components().getComponent("com.yahoo.vespa.hosted.controller.ConfigServerClientMock");
+ configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE, null));
+
+ // POST (deploy) an application with an invalid application package
+ HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty());
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy",
+ entity,
+ Request.Method.POST,
+ athensUserDomain, "mytenant"),
+ new File("deploy-failure.json"), 400);
+
+ // POST (deploy) an application without available capacity
+ configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null));
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy",
+ entity,
+ Request.Method.POST,
+ athensUserDomain, "mytenant"),
+ new File("deploy-out-of-capacity.json"), 400);
+
+ // DELETE tenant which has an application
+ tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE),
+ "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not delete tenant 'tenant1': This tenant has active applications\"}",
+ 400);
+
+ // DELETE application
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.DELETE),
+ "");
+ // DELETE application again - should produce 404
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.DELETE),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}",
+ 404);
+ // DELETE tenant
+ tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE),
+ new File("tenant-without-applications.json"));
+ // DELETE tenant again - should produce 404
+ tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete tenant 'tenant1': Tenant not found\"}",
+ 404);
+
+ // Promote application chef env for nonexistent tenant/application
+ tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST),
+ "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Unable to promote Chef environments for application\"}",
+ 500);
+ }
+
+ @Test
+ public void testAuthorization() throws IOException, URISyntaxException {
+ ContainerTester tester = new ContainerTester(container, responseFiles);
+ String authorizedUser = "mytenant";
+ String unauthorizedUser = "othertenant";
+
+ // Mutation without an authorized user is disallowed
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST,
+ "domain1", null),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"User is not authenticated\"}",
+ 403);
+
+ // ... but read methods are allowed
+ tester.assertResponse(request("/application/v4/tenant/",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.GET,
+ "domain1", null),
+ "[]",
+ 200);
+
+ addTenantAthensDomain("domain1", "mytenant");
+
+ // Creating a tenant for an Athens domain the user is not admin for is disallowed
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST,
+ "domain1", unauthorizedUser),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"The user 'othertenant' is not admin in Athens domain 'domain1'\"}",
+ 403);
+
+ // (Create it with the right tenant id)
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.POST,
+ "domain1", authorizedUser),
+ new File("tenant-without-applications.json"),
+ 200);
+
+ // Creating an application for an Athens domain the user is not admin for is disallowed
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.POST,
+ "domain1", unauthorizedUser),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}",
+ 403);
+
+ // (Create it with the right tenant id)
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.POST,
+ "domain1", authorizedUser),
+ new File("application-reference.json"),
+ 200);
+
+ // Deploy to an authorized zone by a user tenant is disallowed
+ HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty());
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy",
+ entity,
+ Request.Method.POST,
+ athensUserDomain, "mytenant"),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"Principal 'mytenant' is not a screwdriver principal, and does not have deploy access to application 'tenant1.application1'\"}",
+ 403);
+
+ // Deleting an application for an Athens domain the user is not admin for is disallowed
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.DELETE,
+ "domain1", unauthorizedUser),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}",
+ 403);
+
+ // (Deleting it with the right tenant id)
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+ "",
+ Request.Method.DELETE,
+ "domain1", authorizedUser),
+ "",
+ 200);
+
+ // Updating a tenant for an Athens domain the user is not admin for is disallowed
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+ Request.Method.PUT,
+ "domain1", unauthorizedUser),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}",
+ 403);
+
+ // Change Athens domain
+ addTenantAthensDomain("domain2", "mytenant");
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "{\"athensDomain\":\"domain2\", \"property\":\"property1\"}",
+ Request.Method.PUT,
+ "domain1", authorizedUser),
+ "{\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}",
+ 200);
+
+ // Deleting a tenant for an Athens domain the user is not admin for is disallowed
+ tester.assertResponse(request("/application/v4/tenant/tenant1",
+ "",
+ Request.Method.DELETE,
+ "domain1", unauthorizedUser),
+ "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}",
+ 403);
+ }
+
+ private HttpEntity createApplicationDeployData(ApplicationPackage applicationPackage, Optional<Long> screwdriverJobId) {
+ MultipartEntityBuilder builder = MultipartEntityBuilder.create();
+ builder.addTextBody("deployOptions", deployOptions(screwdriverJobId), ContentType.APPLICATION_JSON);
+ builder.addBinaryBody("applicationZip", applicationPackage.zippedContent());
+ return builder.build();
+ }
+
+ private String deployOptions(Optional<Long> screwdriverJobId) {
+ if (screwdriverJobId.isPresent()) // deployment from screwdriver
+ return "{\"vespaVersion\":null," +
+ "\"ignoreValidationErrors\":false," +
+ "\"screwdriverBuildJob\":{\"screwdriverId\":\"" + screwdriverJobId.get() + "\"," +
+ "\"gitRevision\":{\"repository\":\"repository1\"," +
+ "\"branch\":\"master\"," +
+ "\"commit\":\"commit1\"" +
+ "}" +
+ "}" +
+ "}";
+ else // This is ugly and evil, but tentatively replicates the existing behavor from the client on user deployments
+ return "{\"vespaVersion\":null," +
+ "\"ignoreValidationErrors\":false," +
+ "\"screwdriverBuildJob\":{\"screwdriverId\":null," +
+ "\"gitRevision\":{\"repository\":null," +
+ "\"branch\":null," +
+ "\"commit\":null" +
+ "}" +
+ "}" +
+ "}";
+
+ }
+
+ /** Make a request with (athens) user domain1.mytenant1 */
+ private Request request(String path, String data, Request.Method method) {
+ return request(path, data.getBytes(StandardCharsets.UTF_8), method, "domain1", "mytenant", "application/json");
+ }
+
+ private Request request(String path, String data, Request.Method method, String domain, String user) {
+ return request(path, data.getBytes(StandardCharsets.UTF_8), method, domain, user, "application/json");
+ }
+
+ private Request request(String path, byte[] data, Request.Method method, String domain, String user, String contentType) {
+ // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters
+ Request request = new Request("http://localhost:8080" + path + "?domain=" + domain +
+ (user != null ? "&user=" + user : ""),
+ data, method);
+ request.getHeaders().put("Content-Type", contentType);
+ return request;
+ }
+
+ private Request request(String path, HttpEntity data, Request.Method method, String domain, String user) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try {
+ data.writeTo(out);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return request(path, out.toByteArray(), method, domain, user, data.getContentType().getValue());
+ }
+
+ /**
+ * In production this happens outside hosted Vespa, so there is no API for it and we need to reach down into the
+ * mock setup to replicate the action.
+ */
+ private AthensDomain addTenantAthensDomain(String domainName, String userName) {
+ Athens athens = (AthensMock) container.components().getComponent(
+ "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock"
+ );
+ ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory();
+ AthensDomain athensDomain = new AthensDomain(domainName);
+ AthensDbMock.Domain domain = new AthensDbMock.Domain(athensDomain);
+ domain.markAsVespaTenant();
+ domain.admin(new AthensPrincipal(new AthensDomain(athensUserDomain), new UserId(userName)));
+ mock.getSetup().addDomain(domain);
+ return athensDomain;
+ }
+
+ /**
+ * In production this happens outside hosted Vespa, so there is no API for it and we need to reach down into the
+ * mock setup to replicate the action.
+ */
+ private void addScrewdriverUserToDomain(String screwdriverUserId, String domainName) {
+ Athens athens = (AthensMock) container.components().getComponent(
+ "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock"
+ );
+ ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory();
+ AthensDbMock.Domain domain = mock.getSetup().domains.get(new AthensDomain(domainName));
+ domain.admin(new AthensPrincipal(new AthensDomain(athensScrewdriverDomain), new UserId(screwdriverUserId)));
+ }
+
+ private void addMockObservedApplicationCost(String tenant, String application, String instance) {
+ MockInsightBackend mock = (MockInsightBackend) container.components().getComponent("com.yahoo.vespa.hosted.controller.cost.MockInsightBackend");
+
+ ClusterCost cost = new ClusterCost();
+ cost.setCount(2);
+ cost.setResource("cpu");
+ cost.setUtilization(1.0f);
+ cost.setTco(25);
+ cost.setFlavor("flavor1");
+ cost.setWaste(10);
+ cost.setType("content");
+ List<String> hostnames = new ArrayList<>();
+ hostnames.add("host1");
+ hostnames.add("host2");
+ cost.setHostnames(hostnames);
+ Map<String, ClusterCost> clusterCosts = new HashMap<>();
+ clusterCosts.put("cluster1", cost);
+
+ mock.setApplicationCost(new ApplicationId.Builder().tenant(tenant).applicationName(application).instanceName(instance).build(),
+ new ApplicationCost("prod.us-west-1", tenant, application + "." + instance, 37, 1.0f, 0.0f, clusterCosts));
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java
new file mode 100644
index 00000000000..16557157cf5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java
@@ -0,0 +1,78 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.TestIdentities;
+import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain;
+import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken;
+import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService;
+
+import javax.ws.rs.core.SecurityContext;
+import java.security.Principal;
+import java.util.Optional;
+
+/**
+ * This overrides methods in Authorizer which relies on properties set by jdisc HTTP filters.
+ * This is necessary because filters are not currently executed when executing requests with Application.
+ *
+ * @author bratseth
+ */
+@SuppressWarnings("unused") // injected
+public class MockAuthorizer extends Authorizer {
+
+ public MockAuthorizer(Controller controller, EntityService entityService) {
+ super(controller, entityService);
+ }
+
+ /** Returns a principal given by the request parameters 'domain' and 'user' */
+ @Override
+ public Optional<Principal> getPrincipalIfAny(HttpRequest request) {
+ if (request.getProperty("user") == null) return Optional.empty();
+ return Optional.of(new AthensPrincipal(new AthensDomain(request.getProperty("domain")),
+ new UserId(request.getProperty("user"))));
+ }
+
+ /** Returns the hardcoded NToken of {@link TestIdentities#userId} */
+ @Override
+ public Optional<NToken> getNToken(HttpRequest request) {
+ return Optional.of(TestIdentities.userNToken);
+ }
+
+ private static class MockPrincipal implements Principal {
+
+ @Override
+ public String getName() { return TestIdentities.userId.id(); }
+
+ }
+
+ @Override
+ protected Optional<SecurityContext> securityContextOf(HttpRequest request) {
+ return getPrincipalIfAny(request).map(MockSecurityContext::new);
+ }
+
+ private static final class MockSecurityContext implements SecurityContext {
+
+ private final Principal principal;
+
+ private MockSecurityContext(Principal principal) {
+ this.principal = principal;
+ }
+
+ @Override
+ public Principal getUserPrincipal() { return principal; }
+
+ @Override
+ public boolean isUserInRole(String role) { return false; }
+
+ @Override
+ public boolean isSecure() { return true; }
+
+ @Override
+ public String getAuthenticationScheme() { throw new UnsupportedOperationException(); }
+
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java
new file mode 100644
index 00000000000..1a623c4e3eb
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParserTest.java
@@ -0,0 +1,91 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.google.inject.Key;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.jdisc.Container;
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.ResourceReference;
+import com.yahoo.jdisc.handler.RequestHandler;
+import com.yahoo.jdisc.service.CurrentContainer;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class MultipartParserTest {
+
+ @Test
+ public void multipartParserTest() throws URISyntaxException {
+ String data =
+ "Content-Type: multipart/form-data; boundary=AaB03x\r\n" +
+ "\r\n" +
+ "--AaB03x\r\n" +
+ "Content-Disposition: form-data; name=\"submit-name\"\r\n" +
+ "\r\n" +
+ "Larry\r\n" +
+ "--AaB03x\r\n" +
+ "Content-Disposition: form-data; name=\"submit-address\"\r\n" +
+ "Content-Type: text/plain\r\n" +
+ "\r\n" +
+ "House 1\r\n" +
+ "--AaB03x\r\n" +
+ "Content-Disposition: form-data; name=\"files\"; filename=\"file1.txt\"\r\n" +
+ "Content-Type: text/plain\r\n" +
+ "\r\n" +
+ "... contents of file1.txt ...\r\n" +
+ "--AaB03x--\r\n";
+ ByteArrayInputStream dataStream = new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8));
+ HttpRequest request = HttpRequest.createRequest(new MockCurrentContainer(),
+ new URI("http://foo"),
+ com.yahoo.jdisc.http.HttpRequest.Method.POST,
+ dataStream);
+ request.getJDiscRequest().headers().put("Content-Type", "multipart/form-data; boundary=AaB03x");
+ Map<String, byte[]> parts = new MultipartParser().parse(request);
+ assertEquals(3, parts.size());
+ assertTrue(parts.containsKey("submit-name"));
+ assertTrue(parts.containsKey("submit-address"));
+ assertTrue(parts.containsKey("files"));
+ assertEquals("Larry", new String(parts.get("submit-name"), StandardCharsets.UTF_8));
+ assertEquals("... contents of file1.txt ...", new String(parts.get("files"), StandardCharsets.UTF_8));
+ }
+
+ private static class MockCurrentContainer implements CurrentContainer {
+
+ @Override
+ public Container newReference(URI uri) { return new MockContainer(); }
+
+ }
+
+ private static class MockContainer implements Container {
+
+ @Override
+ public RequestHandler resolveHandler(Request request) { return null; }
+
+ @Override
+ public <T> T getInstance(Key<T> key) { return null; }
+
+ @Override
+ public <T> T getInstance(Class<T> aClass) { return null; }
+
+ @Override
+ public ResourceReference refer() { return null; }
+
+ @Override
+ public void release() { }
+
+ @Override
+ public long currentTimeMillis() { return 0; }
+
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java
new file mode 100644
index 00000000000..7b1d8d17a5c
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java
@@ -0,0 +1,63 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.vespa.hosted.controller.restapi.Path;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class PathTest {
+
+ @Test
+ public void testPath() {
+ assertFalse(new Path("").matches("/a/{foo}/bar/{b}"));;
+ assertFalse(new Path("///").matches("/a/{foo}/bar/{b}"));;
+ assertFalse(new Path("///foo").matches("/a/{foo}/bar/{b}"));;
+ assertFalse(new Path("///bar/").matches("/a/{foo}/bar/{b}"));;
+ Path path = new Path("/a/1/bar/fuz");
+ assertTrue(path.matches("/a/{foo}/bar/{b}"));;
+ assertEquals("1", path.get("foo"));
+ assertEquals("fuz", path.get("b"));
+ }
+
+ @Test
+ public void testPathWithRest() {
+ {
+ Path path = new Path("/a/1/bar/fuz/");
+ assertTrue(path.matches("/a/{foo}/bar/{b}/{*}"));
+ assertEquals("1", path.get("foo"));
+ assertEquals("fuz", path.get("b"));
+ assertEquals("", path.getRest());
+ }
+
+ {
+ Path path = new Path("/a/1/bar/fuz/kanoo");
+ assertTrue(path.matches("/a/{foo}/bar/{b}/{*}"));
+ assertEquals("1", path.get("foo"));
+ assertEquals("fuz", path.get("b"));
+ assertEquals("kanoo", path.getRest());
+ }
+
+ {
+ Path path = new Path("/a/1/bar/fuz/kanoo/trips");
+ assertTrue(path.matches("/a/{foo}/bar/{b}/{*}"));
+ assertEquals("1", path.get("foo"));
+ assertEquals("fuz", path.get("b"));
+ assertEquals("kanoo/trips", path.getRest());
+ }
+
+ {
+ Path path = new Path("/a/1/bar/fuz/kanoo/trips/");
+ assertTrue(path.matches("/a/{foo}/bar/{b}/{*}"));
+ assertEquals("1", path.get("foo"));
+ assertEquals("fuz", path.get("b"));
+ assertEquals("kanoo/trips/", path.getRest());
+ }
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java
new file mode 100644
index 00000000000..6cf90905679
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java
@@ -0,0 +1,95 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.serviceview.bindings.ApplicationView;
+import com.yahoo.vespa.serviceview.bindings.ClusterView;
+import com.yahoo.vespa.serviceview.bindings.ServiceView;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class ServiceApiResponseTest {
+
+ private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/";
+
+ @Test
+ public void testServiceViewResponse() throws URISyntaxException, IOException {
+ ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.prod, RegionName.from("us-west-1")),
+ ApplicationId.from("tenant1", "application1", "default"),
+ Collections.singletonList(new URI("config-server1")),
+ new URI("http://server1:4080/request/path?foo=bar"));
+ ApplicationView applicationView = new ApplicationView();
+ ClusterView clusterView = new ClusterView();
+ clusterView.type = "container";
+ clusterView.name = "cluster1";
+ clusterView.url = "cluster-url";
+ ServiceView serviceView = new ServiceView();
+ serviceView.url = null;
+ serviceView.serviceType = "container";
+ serviceView.serviceName = "service1";
+ serviceView.configId = "configId1";
+ serviceView.host = "host1";
+ serviceView.legacyStatusPages = "legacyPages";
+ clusterView.services = Collections.singletonList(serviceView);
+ applicationView.clusters = Collections.singletonList(clusterView);
+ response.setResponse(applicationView);
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ response.render(stream);
+ Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray());
+ Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response.json")).getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("service-api-response.json",
+ new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8),
+ new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testServiceViewResponseWithURLs() throws URISyntaxException, IOException {
+ ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.prod, RegionName.from("us-west-1")),
+ ApplicationId.from("tenant2", "application2", "default"),
+ Collections.singletonList(new URI("http://cfg1.test/")),
+ new URI("http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1"));
+ ApplicationView applicationView = new ApplicationView();
+ ClusterView clusterView = new ClusterView();
+ clusterView.type = "container";
+ clusterView.name = "cluster1";
+ clusterView.url = "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health";
+ ServiceView serviceView = new ServiceView();
+ serviceView.url = null;
+ serviceView.serviceType = "container";
+ serviceView.serviceName = "service1";
+ serviceView.configId = "configId1";
+ serviceView.host = "host1";
+ serviceView.legacyStatusPages = "legacyPages";
+ clusterView.services = Collections.singletonList(serviceView);
+ applicationView.clusters = Collections.singletonList(clusterView);
+ response.setResponse(applicationView);
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ response.render(stream);
+ Slime responseSlime = SlimeUtils.jsonToSlime(stream.toByteArray());
+ Slime expectedSlime = SlimeUtils.jsonToSlime(IOUtils.readFile(new File(responseFiles + "service-api-response-with-urls.json")).getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("service-api-response.json",
+ new String(SlimeUtils.toJsonBytes(expectedSlime), StandardCharsets.UTF_8),
+ new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8));
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json
new file mode 100644
index 00000000000..f74b2290d81
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json
@@ -0,0 +1 @@
+{"message":"Triggered deployment of application 'tenant1.application1' on version 6.1"} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json
new file mode 100644
index 00000000000..ecee1c8dbde
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json
@@ -0,0 +1,3 @@
+[
+ @include(application-reference.json)
+] \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json
new file mode 100644
index 00000000000..1ec229a2b4a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json
@@ -0,0 +1,5 @@
+{
+ "application":"application1",
+ "instance":"default",
+ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1"
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
new file mode 100644
index 00000000000..5df690f5bc7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
@@ -0,0 +1,104 @@
+{
+ "deploying": {
+ "version": "(ignore)"
+ },
+ "deploymentJobs": [
+ {
+ "type": "system-test",
+ "success": true,
+ "lastTriggered": {
+ "version": "(ignore)",
+ "at": "(ignore)"
+ },
+ "lastCompleted": {
+ "version": "(ignore)",
+ "at": "(ignore)"
+ },
+ "lastSuccess": {
+ "version": "(ignore)",
+ "at": "(ignore)"
+ }
+ },
+ {
+ "type":"staging-test",
+ "success":true,
+ "lastTriggered":{
+ "version":"(ignore)",
+ "at":"(ignore)"
+ },
+ "lastCompleted":{
+ "version":"(ignore)",
+ "at":"(ignore)"
+ },
+ "lastSuccess":{
+ "version":"(ignore)",
+ "at":"(ignore)"
+ }
+ },
+ {
+ "type":"production-corp-us-east-1",
+ "success":false,
+ "lastTriggered":{
+ "version":"(ignore)",
+ "revision":{
+ "hash":"(ignore)",
+ "source":{
+ "gitRepository":"repository1",
+ "gitBranch":"master",
+ "gitCommit":"commit1"
+ }
+ },
+ "at":"(ignore)"
+ },
+ "lastCompleted":{
+ "version":"(ignore)",
+ "revision":{
+ "hash":"(ignore)",
+ "source":{
+ "gitRepository":"repository1",
+ "gitBranch":"master",
+ "gitCommit":"commit1"
+ }
+ },
+ "at":"(ignore)"
+ },
+ "firstFailing":{
+ "version":"(ignore)",
+ "revision":{
+ "hash":"(ignore)",
+ "source":{
+ "gitRepository":"repository1",
+ "gitBranch":"master",
+ "gitCommit":"commit1"
+ }
+ },
+ "at":"(ignore)"
+ }
+ }
+ ],
+ "compileVersion": "(ignore)",
+ "globalRotations": [
+ "http://fake-global-rotation-tenant1.application1"
+ ],
+ "instances": [
+ {
+ "environment": "dev",
+ "region": "us-west-1",
+ "instance": "default",
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"
+ },
+ {
+ "environment": "prod",
+ "region": "corp-us-east-1",
+ "instance": "default",
+ "bcpStatus": {"rotationStatus":"UNKNOWN"},
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"
+ }
+ ]
+ ,
+ "metrics":
+ {
+ "queryServiceQuality":0.5,
+ "writeServiceQuality":0.7
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json
new file mode 100644
index 00000000000..3a1cc9c6582
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/athensDomain-list.json
@@ -0,0 +1,5 @@
+{
+ "data": [
+ "domain1"
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json
new file mode 100644
index 00000000000..acfb67b702b
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/convergence.json
@@ -0,0 +1,3 @@
+{
+ "generation": 1
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json
new file mode 100644
index 00000000000..3c428332aa6
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/cookiefreshness.json
@@ -0,0 +1,3 @@
+{
+ "shouldRefreshCookie":true
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json
new file mode 100644
index 00000000000..709548e87a7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/create-user-response.json
@@ -0,0 +1,3 @@
+{
+ "message":"Created user 'newuser'"
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-tenant-response.json
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json
new file mode 100644
index 00000000000..f9496edecde
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-error-result.json
@@ -0,0 +1,4 @@
+{
+ "error-code":"BAD_REQUEST",
+ "message":"Failed to prepare application"
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json
new file mode 100644
index 00000000000..0de6465156f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-failure.json
@@ -0,0 +1,4 @@
+{
+ "error-code":"INVALID_APPLICATION_PACKAGE",
+ "message":"Failed to prepare application"
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json
new file mode 100644
index 00000000000..669df626378
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-out-of-capacity.json
@@ -0,0 +1,4 @@
+{
+ "error-code":"OUT_OF_CAPACITY",
+ "message":"Failed to prepare application"
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json
new file mode 100644
index 00000000000..d1ae5253a00
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json
@@ -0,0 +1,9 @@
+{
+ "revisionId":"(ignore)",
+ "applicationZipSize":412,
+ "prepareMessages":[],
+ "configChangeActions":{
+ "restart":[],
+ "refeed":[]
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
new file mode 100644
index 00000000000..50599581f92
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
@@ -0,0 +1,57 @@
+{
+ "serviceUrls": [
+ "qrs-endpoint","feeding-endpoint","global-endpoint","alias-endpoint"
+ ],
+ "nodes": "http://localhost:8080/zone/v2/prod/corp-us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default",
+ "elkUrl": "http://log.prod.corp-us-east-1.test/#/discover?_g=()&_a=(columns:!(_source),index:'logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'HV-tenant:%22tenant1%22%20AND%20HV-application:%22application1%22%20AND%20HV-region:%22corp-us-east-1%22%20AND%20HV-instance:%22default%22%20AND%20HV-environment:%22prod%22')),sort:!('@timestamp',desc))",
+ "yamasUrl": "http://monitoring-system.test/?environment=prod&region=corp-us-east-1&application=tenant1.application1",
+ "version": "(ignore)",
+ "revision": "(ignore)",
+ "deployTimeEpochMs": "(ignore)",
+ "screwdriverId":"123",
+ "gitRepository":"repository1",
+ "gitBranch":"master",
+ "gitCommit":"commit1",
+ "cost": {
+ "zone": "prod.us-west-1",
+ "tenant": "tenant1",
+ "app": "application1.default",
+ "tco": 37,
+ "utilization": 1.0,
+ "waste": 0.0,
+ "cluster": {
+ "cluster1": {
+ "count": 2,
+ "resource": "cpu",
+ "utilization": 1.0,
+ "tco": 25,
+ "flavor": "flavor1",
+ "waste": 10,
+ "type": "content",
+ "util": {
+ "cpu": 0.0,
+ "mem": 0.0,
+ "disk": 0.0,
+ "diskBusy": 0.0
+ },
+ "usage": {
+ "cpu": 0.0,
+ "mem": 0.0,
+ "disk": 0.0,
+ "diskBusy": 0.0
+ },
+ "hostnames": [
+ "host1",
+ "host2"
+ ]
+ }
+ }
+ },
+ "metrics": {
+ "queriesPerSecond":1.0,
+ "writesPerSecond":2.0,
+ "documentCount":3.0,
+ "queryLatencyMillis":4.0,
+ "writeLatencyMillis":5.0
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json
new file mode 100644
index 00000000000..1ed86920fdf
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json
@@ -0,0 +1 @@
+{"message":"Rotations [global-endpoint, alias-endpoint] successfully set to in service"} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
new file mode 100644
index 00000000000..31a337f7706
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
@@ -0,0 +1 @@
+{"globalrotationoverride":["global-endpoint",{"status":"in","reason":"","agent":"","timestamp":1497618757},"alias-endpoint",{"status":"in","reason":"","agent":"","timestamp":1497618757}]} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json
new file mode 100644
index 00000000000..eefd2859241
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json
@@ -0,0 +1 @@
+{"message":"Rotations [global-endpoint, alias-endpoint] successfully set to out of service"} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json
new file mode 100644
index 00000000000..530e21c6c7a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation.json
@@ -0,0 +1,5 @@
+{
+ "bcpStatus": {
+ "rotationStatus": "IN"
+ }
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json
new file mode 100644
index 00000000000..9e26dfeeb6e
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/log-response.json
@@ -0,0 +1 @@
+{} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json
new file mode 100644
index 00000000000..8de85754ab0
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json
@@ -0,0 +1,9 @@
+{
+ "type": "OPSDB",
+ "property": "property1",
+ "propertyId": "1234",
+ "userGroup": "group1",
+ "applications": [
+
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json
new file mode 100644
index 00000000000..9f0a7ec603e
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json
@@ -0,0 +1,9 @@
+{
+ "type": "OPSDB",
+ "property": "property2",
+ "propertyId": "4321",
+ "userGroup": "group1",
+ "applications": [
+
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json
new file mode 100644
index 00000000000..596dea037bd
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/property-list.json
@@ -0,0 +1,6 @@
+{
+ "properties": [
+ {"propertyid": "1234", "property": "foo"},
+ {"propertyid": "4321", "property": "bar"}
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json
new file mode 100644
index 00000000000..6e4e319d3e1
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json
@@ -0,0 +1,22 @@
+{
+ "resources":[
+ {
+ "url":"http://localhost:8080/application/v4/user/"
+ },
+ {
+ "url":"http://localhost:8080/application/v4/tenant/"
+ },
+ {
+ "url":"http://localhost:8080/application/v4/tenant-pipeline/"
+ },
+ {
+ "url":"http://localhost:8080/application/v4/athensDomain/"
+ },
+ {
+ "url":"http://localhost:8080/application/v4/property/"
+ },
+ {
+ "url":"http://localhost:8080/application/v4/cookiefreshness/"
+ }
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json
new file mode 100644
index 00000000000..0e610c4d4b2
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response-with-urls.json
@@ -0,0 +1,18 @@
+{
+ "clusters": [
+ {
+ "name": "cluster1",
+ "type": "container",
+ "url": "http://cfg1.test/serviceview/v1/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/service/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/searchnode-9dujk1pa0vufxrj6n4yvmi8uc/state/v1/health",
+ "services": [
+ {
+ "url": null,
+ "serviceType": "container",
+ "serviceName": "service1",
+ "configId": "configId1",
+ "host": "host1"
+ }
+ ]
+ }
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json
new file mode 100644
index 00000000000..3380eb26911
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service-api-response.json
@@ -0,0 +1,18 @@
+{
+ "clusters": [
+ {
+ "name": "cluster1",
+ "type": "container",
+ "url": "cluster-url",
+ "services": [
+ {
+ "url": null,
+ "serviceType": "container",
+ "serviceName": "service1",
+ "configId": "configId1",
+ "host": "host1"
+ }
+ ]
+ }
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json
new file mode 100644
index 00000000000..8fb64d65ff8
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/service.json
@@ -0,0 +1,7 @@
+{
+ "resources": [
+ {
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/filedistributorservice-dud1f4w037qdxdrn0ovxfdtgw/state/v1/config"
+ }
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json
new file mode 100644
index 00000000000..8a0849393c9
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json
@@ -0,0 +1,18 @@
+{
+ "clusters": [
+ {
+ "name": "cluster1",
+ "type": "content",
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1",
+ "services": [
+ {
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/",
+ "serviceType": "storagenode",
+ "serviceName": "storagenode",
+ "configId": "cluster1/storage/0",
+ "host": "host1"
+ }
+ ]
+ }
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json
new file mode 100644
index 00000000000..a9d9cd33ae8
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-list.json
@@ -0,0 +1,11 @@
+[
+ {
+ "tenant": "tenant1",
+ "metaData": {
+ "type": "ATHENS",
+ "athensDomain": "domain1",
+ "property": "property1"
+ },
+ "url": "http://localhost:8080/application/v4/tenant/tenant1"
+ }
+] \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json
new file mode 100644
index 00000000000..4e6fef1b994
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-pipelines.json
@@ -0,0 +1,13 @@
+{
+ "tenantPipelines": [
+ {
+ "screwdriverId": "123",
+ "tenant": "tenant1",
+ "application": "application1",
+ "instance": "default"
+ }
+ ],
+ "brokenTenantPipelines": [
+
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json
new file mode 100644
index 00000000000..87901218c2e
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json
@@ -0,0 +1,12 @@
+{
+ "type": "ATHENS",
+ "athensDomain": "domain1",
+ "property": "property1",
+ "applications": [
+ {
+ "application":"application1",
+ "instance":"default",
+ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1"
+ }
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json
new file mode 100644
index 00000000000..3deef01bb44
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json
@@ -0,0 +1,9 @@
+{
+ "type": "ATHENS",
+ "athensDomain": "domain2",
+ "property": "property2",
+ "propertyId": "1234",
+ "applications": [
+
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json
new file mode 100644
index 00000000000..88ec5ec7d3d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json
@@ -0,0 +1,8 @@
+{
+ "type": "ATHENS",
+ "athensDomain": "domain1",
+ "property": "property1",
+ "applications": [
+
+ ]
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json
new file mode 100644
index 00000000000..d3927cbcfcf
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json
@@ -0,0 +1,5 @@
+{
+ "user": "mytenant",
+ "tenants": @include(tenant-list.json),
+ "tenantExists": false
+} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
new file mode 100644
index 00000000000..011bbadb91c
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -0,0 +1,40 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * @author bratseth
+ */
+public class ControllerApiTest extends ControllerContainerTest {
+
+ private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/";
+
+ @Test
+ public void testControllerApi() throws IOException {
+ ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+
+ tester.assertResponse(new Request("http://localhost:8080/controller/v1/"), new File("root.json"));
+
+ // POST deactivation of a maintenance job
+ assertResponse(new Request("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+ new byte[0], Request.Method.POST),
+ 200,
+ "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}");
+ // GET a list of all maintenance jobs
+ tester.assertResponse(new Request("http://localhost:8080/controller/v1/maintenance/"),
+ new File("maintenance.json"));
+ // DELETE deactivation of a maintenance job
+ assertResponse(new Request("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer",
+ new byte[0], Request.Method.DELETE),
+ 200,
+ "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}");
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
new file mode 100644
index 00000000000..d8ca5e59b4f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -0,0 +1,31 @@
+{
+ "jobs":[
+ {
+ "name":"DelayedDeployer"
+ },
+ {
+ "name":"Upgrader"
+ },
+ {
+ "name":"FailureRedeployer"
+ },
+ {
+ "name":"DeploymentExpirer"
+ },
+ {
+ "name":"MetricsReporter"
+ },
+ {
+ "name":"VersionStatusUpdater"
+ },
+ {
+ "name":"DeploymentIssueReporter"
+ },
+ {
+ "name":"OutstandingChangeDeployer"
+ }
+ ],
+ "inactive":[
+ "DeploymentExpirer"
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json
new file mode 100644
index 00000000000..155c13fd5ed
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/root.json
@@ -0,0 +1,7 @@
+{
+ "resources":[
+ {
+ "url":"http://localhost:8080/controller/v1/maintenance/"
+ }
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
new file mode 100644
index 00000000000..26741148d3a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -0,0 +1,72 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.deployment;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
+
+/**
+ * @author bratseth
+ */
+public class DeploymentApiTest extends ControllerContainerTest {
+
+ private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/";
+
+ @Test
+ public void testDeploymentApi() throws IOException {
+ ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+ tester.containerTester().updateSystemVersion();
+ long projectId = 11;
+ Application app = tester.createApplication();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+ tester.notifyJobCompletion(app.id(), projectId, true, component);
+ tester.deploy(app, applicationPackage, new Zone(Environment.test, RegionName.from("us-east-1")), projectId);
+ tester.notifyJobCompletion(app.id(), projectId, true, systemTest);
+ tester.deploy(app, applicationPackage, new Zone(Environment.staging, RegionName.from("us-east-3")), projectId);
+ tester.notifyJobCompletion(app.id(), projectId, false, stagingTest);
+
+ tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()),
+ tester.controller()));
+ tester.assertResponse(new Request("http://localhost:8080/deployment/v1/"),
+ new File("root.json"));
+ }
+
+ private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) {
+ List<VespaVersion> censored = new ArrayList<>();
+ for (VespaVersion version : versionStatus.versions()) {
+ if ( ! version.configServerHostnames().isEmpty())
+ version = new VespaVersion(version.statistics(),
+ version.releaseCommit(),
+ version.releasedAt(),
+ version.isCurrentSystemVersion(),
+ ImmutableSet.of("config1.test", "config2.test"),
+ controller);
+ censored.add(version);
+ }
+ return new VersionStatus(censored);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
new file mode 100644
index 00000000000..4ea1359519f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -0,0 +1,50 @@
+{
+ "versions":[
+ {
+ "version":"(ignore)",
+ "confidence":"normal",
+ "commit":"(ignore)",
+ "date":0,
+ "controllerVersion":false,
+ "systemVersion":true,
+ "configServers":[
+ {
+ "hostname":"config1.test"
+ },
+ {
+ "hostname":"config2.test"
+ }
+ ],
+ "failingApplications":[
+ {
+ "tenant":"tenant1",
+ "application":"application1",
+ "instance":"default",
+ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+ "failingSince": "(ignore)"
+ }
+ ],
+ "productionApplications":[
+
+ ]
+ },
+ {
+ "version":"(ignore)",
+ "confidence":"normal",
+ "commit":"(ignore)",
+ "date":0,
+ "controllerVersion":true,
+ "systemVersion":false,
+ "configServers":[
+
+ ],
+ "failingApplications":[
+
+ ],
+ "productionApplications":[
+
+ ]
+ }
+ ]
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java
new file mode 100644
index 00000000000..0c31c6e2cc5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlRequestFilterTest.java
@@ -0,0 +1,79 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.yahoo.jdisc.HeaderFields;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.ContentChannel;
+import com.yahoo.jdisc.handler.ResponseHandler;
+import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig;
+import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig.Builder;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static com.yahoo.jdisc.http.HttpRequest.Method.OPTIONS;
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS;
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author gjoranv
+ */
+public class AccessControlRequestFilterTest {
+
+ @Test
+ public void any_options_request_yields_access_control_headers_in_response() {
+ HeaderFields headers = doFilterRequest(newRequestFilter(), "http://any.origin");
+ ACCESS_CONTROL_HEADERS.keySet().forEach(
+ header -> assertFalse("Empty header: " + header, headers.getFirst(header).isEmpty()));
+ }
+
+ @Test
+ public void allowed_request_origin_yields_allow_origin_header_in_response() {
+ final String ALLOWED_ORIGIN = "http://allowed.origin";
+ HeaderFields headers = doFilterRequest(newRequestFilter(ALLOWED_ORIGIN), ALLOWED_ORIGIN);
+ assertEquals(ALLOWED_ORIGIN, headers.getFirst(ALLOW_ORIGIN_HEADER));
+ }
+
+ @Test
+ public void disallowed_request_origin_does_not_yield_allow_origin_header_in_response() {
+ HeaderFields headers = doFilterRequest(newRequestFilter("http://allowed.origin"), "http://disallowed.origin");
+ assertNull(headers.getFirst(ALLOW_ORIGIN_HEADER));
+ }
+
+ private static HeaderFields doFilterRequest(SecurityRequestFilter filter, String originUrl) {
+ AccessControlResponseHandler responseHandler = new AccessControlResponseHandler();
+ filter.filter(newOptionsRequest(originUrl), responseHandler);
+ return responseHandler.response.headers();
+ }
+
+ private static DiscFilterRequest newOptionsRequest(String origin) {
+ DiscFilterRequest request = mock(DiscFilterRequest.class);
+ when(request.getHeader("Origin")).thenReturn(origin);
+ when(request.getMethod()).thenReturn(OPTIONS.name());
+ return request;
+ }
+
+ private static AccessControlRequestFilter newRequestFilter(String... allowedOriginUrls) {
+ Builder builder = new Builder();
+ Arrays.asList(allowedOriginUrls).forEach(builder::allowedUrls);
+ return new AccessControlRequestFilter(new HttpAccessControlConfig(builder));
+ }
+
+ private static class AccessControlResponseHandler implements ResponseHandler {
+ Response response;
+
+ @Override
+ public ContentChannel handleResponse(Response response) {
+ this.response = response;
+ return mock(ContentChannel.class);
+ }
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java
new file mode 100644
index 00000000000..1b368d0a4b8
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AccessControlResponseFilterTest.java
@@ -0,0 +1,112 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.filter;
+
+import com.yahoo.jdisc.http.Cookie;
+import com.yahoo.jdisc.http.filter.DiscFilterResponse;
+import com.yahoo.jdisc.http.filter.RequestView;
+import com.yahoo.jdisc.http.filter.SecurityResponseFilter;
+import com.yahoo.jdisc.http.servlet.ServletOrJdiscHttpResponse;
+import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig;
+import com.yahoo.vespa.hosted.controller.restapi.filter.config.HttpAccessControlConfig.Builder;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ACCESS_CONTROL_HEADERS;
+import static com.yahoo.vespa.hosted.controller.restapi.filter.AccessControlHeaders.ALLOW_ORIGIN_HEADER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author gjoranv
+ */
+public class AccessControlResponseFilterTest {
+
+ @Test
+ public void any_request_yields_access_control_headers_in_response() {
+ Map<String, String> headers = doFilterRequest(newResponseFilter(), "http://any.origin");
+ ACCESS_CONTROL_HEADERS.keySet().forEach(
+ header -> assertFalse("Empty header: " + header, headers.get(header).isEmpty()));
+ }
+
+ @Test
+ public void allowed_request_origin_yields_allow_origin_header_in_response() {
+ final String ALLOWED_ORIGIN = "http://allowed.origin";
+ Map<String, String> headers = doFilterRequest(newResponseFilter(ALLOWED_ORIGIN), ALLOWED_ORIGIN);
+ assertEquals(ALLOWED_ORIGIN, headers.get(ALLOW_ORIGIN_HEADER));
+ }
+
+ @Test
+ public void disallowed_request_origin_does_not_yield_allow_origin_header_in_response() {
+ Map<String, String> headers = doFilterRequest(newResponseFilter("http://allowed.origin"), "http://disallowed.origin");
+ assertNull(headers.get(ALLOW_ORIGIN_HEADER));
+ }
+
+ @Test
+ public void any_request_origin_yields_allow_origin_header_in_response_when_wildcard_is_allowed() {
+ Map<String, String> headers = doFilterRequest(newResponseFilter("*"), "http://any.origin");
+ assertEquals("*", headers.get(ALLOW_ORIGIN_HEADER));
+ }
+
+ private static Map<String, String> doFilterRequest(SecurityResponseFilter filter, String originUrl) {
+ TestResponse response = new TestResponse();
+ filter.filter(response, newRequestView(originUrl));
+ return Collections.unmodifiableMap(response.headers);
+ }
+
+ private static AccessControlResponseFilter newResponseFilter(String... allowedOriginUrls) {
+ Builder builder = new Builder();
+ Arrays.asList(allowedOriginUrls).forEach(builder::allowedUrls);
+ return new AccessControlResponseFilter(new HttpAccessControlConfig(builder));
+ }
+
+ private static RequestView newRequestView(String originUrl) {
+ RequestView request = mock(RequestView.class);
+ when(request.getFirstHeader("Origin")).thenReturn(Optional.of(originUrl));
+ return request;
+ }
+
+ private static class TestResponse extends DiscFilterResponse {
+ Map<String, String> headers = new HashMap<>();
+
+ TestResponse() {
+ super(mock(ServletOrJdiscHttpResponse.class));
+ }
+
+ @Override
+ public void setHeader(String name, String value) {
+ headers.put(name, value);
+ }
+
+ @Override
+ public String getHeader(String name) {
+ return headers.get(name);
+ }
+
+ @Override
+ public void removeHeaders(String s) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public void setHeaders(String s, String s1) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public void setHeaders(String s, List<String> list) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public void addHeader(String s, String s1) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public void setCookies(List<Cookie> list) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public void setStatus(int i) { throw new UnsupportedOperationException(); }
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json
new file mode 100644
index 00000000000..90b1b027529
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/root-response.json
@@ -0,0 +1,41 @@
+{
+ "services":[
+ {
+ "name":"provision",
+ "url":"http://localhost:8080/provision/v1/",
+ "wadl":"http://localhost:8080/provision/application.wadl"
+ },
+ {
+ "name":"statuspage",
+ "url":"http://localhost:8080/statuspage/v1/",
+ "wadl":"http://localhost:8080/statuspage/application.wadl"
+ },
+ {
+ "name":"zone",
+ "url":"http://localhost:8080/zone/v1/",
+ "wadl":"http://localhost:8080/zone/application.wadl"
+ },
+ {
+ "name":"zone",
+ "url":"http://localhost:8080/zone/v2/",
+ "wadl":"http://localhost:8080/zone/application.wadl"
+ },
+ {
+ "name":"cost",
+ "url":"http://localhost:8080/cost/v1/",
+ "wadl":"http://localhost:8080/cost/application.wadl"
+ },
+ {
+ "name":"application",
+ "url":"http://localhost:8080/application/v4/"
+ },
+ {
+ "name":"deployment",
+ "url":"http://localhost:8080/deployment/v1/"
+ },
+ {
+ "name":"screwdriver",
+ "url":"http://localhost:8080/screwdriver/v1/release/vespa"
+ }
+ ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java
new file mode 100644
index 00000000000..bdfd0f9794f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java
@@ -0,0 +1,165 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.screwdriver;
+
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.application.container.handler.Response;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ * @author jvenstad
+ */
+public class ScrewdriverApiTest extends ControllerContainerTest {
+
+ private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/";
+ private static final Zone testZone = new Zone(Environment.test, RegionName.from("us-east-1"));
+ private static final Zone stagingZone = new Zone(Environment.staging, RegionName.from("us-east-3"));
+
+ @Test
+ public void testGetReleaseStatus() throws Exception {
+ ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+ tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/release/vespa"),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"Information about the current system version is not available at this time\"}",
+ 404);
+
+ tester.controller().updateVersionStatus(VersionStatus.compute(tester.controller()));
+ tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/release/vespa"),
+ new File("release-response.json"), 200);
+ }
+
+ @Test
+ public void testJobStatusReporting() throws Exception {
+ ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+ tester.containerTester().updateSystemVersion();
+ long projectId = 1;
+ Application app = tester.createApplication();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+
+ Version vespaVersion = new Version("6.1"); // system version from mock config server client
+
+ // Make web service calls.
+ notifyCompletion(app.id(), projectId, JobType.component, Optional.empty());
+ tester.deploy(app, applicationPackage, testZone, projectId);
+ notifyCompletion(app.id(), projectId, JobType.systemTest, Optional.empty());
+
+ // Notifying about unknown job fails
+ tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport",
+ jsonReport(app.id(), JobType.productionUsEast3, projectId, 1L,
+ Optional.empty(), false, true)
+ .getBytes(StandardCharsets.UTF_8),
+ Request.Method.POST),
+ new File("unexpected-completion.json"), 400);
+
+ // ... and assert it was recorded
+ JobStatus recordedStatus =
+ tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(JobType.component);
+
+ assertNotNull("Status was recorded", recordedStatus);
+ assertTrue(recordedStatus.isSuccess());
+ assertEquals(vespaVersion, recordedStatus.lastCompleted().get().version());
+
+ recordedStatus =
+ tester.controller().applications().get(app.id()).get().deploymentJobs().jobStatus().get(JobType.productionApNortheast2);
+ assertNull("Status of never-triggered jobs is empty", recordedStatus);
+
+ Response response;
+
+ response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.GET));
+ assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id()));
+ assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id()));
+ assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries());
+
+ // Check that GET didn't affect the enqueued jobs.
+ response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE));
+ assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id()));
+ assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id()));
+ assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries());
+
+ Thread.sleep(50);
+ // Check that the *first* DELETE has removed the enqueued jobs.
+ assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE),
+ 200, "[]");
+ }
+
+ @Test
+ public void testJobStatusReportingOutOfCapacity() throws Exception {
+ ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+ tester.containerTester().updateSystemVersion();
+
+ long projectId = 1;
+ Application app = tester.createApplication();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .environment(Environment.prod)
+ .region("corp-us-east-1")
+ .build();
+
+ // Report job failing with out of capacity
+ notifyCompletion(app.id(), projectId, JobType.component, Optional.empty());
+ tester.deploy(app, applicationPackage, testZone, projectId);
+ notifyCompletion(app.id(), projectId, JobType.systemTest, Optional.empty());
+ tester.deploy(app, applicationPackage, stagingZone, projectId);
+ notifyCompletion(app.id(), projectId, JobType.stagingTest, Optional.of(JobError.outOfCapacity));
+
+ // Appropriate error is recorded
+ JobStatus jobStatus = tester.controller().applications().get(app.id())
+ .get()
+ .deploymentJobs()
+ .jobStatus()
+ .get(JobType.stagingTest);
+ assertFalse(jobStatus.isSuccess());
+ assertEquals(JobError.outOfCapacity, jobStatus.jobError().get());
+ }
+
+ private void notifyCompletion(ApplicationId app, long projectId, JobType jobType, Optional<JobError> error) throws IOException {
+ assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport",
+ jsonReport(app, jobType, projectId, 1L, error, false, true).getBytes(StandardCharsets.UTF_8),
+ Request.Method.POST),
+ 200, "ok");
+ }
+
+ private static String jsonReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber,
+ Optional<JobError> jobError, boolean selfTriggering, boolean gitChanges) {
+ return
+ "{\n" +
+ " \"projectId\" : " + projectId + ",\n" +
+ " \"jobName\" :\"" + jobType.id() + "\",\n" +
+ " \"buildNumber\" : " + buildNumber + ",\n" +
+ jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") +
+ " \"selfTriggering\": " + selfTriggering + ",\n" +
+ " \"gitChanges\" : " + gitChanges + ",\n" +
+ " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" +
+ " \"application\" :\"" + applicationId.application().value() + "\",\n" +
+ " \"instance\" :\"" + applicationId.instance().value() + "\"\n" +
+ "}";
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json
new file mode 100644
index 00000000000..9d96e08e695
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/release-response.json
@@ -0,0 +1,5 @@
+{
+ "version":"(ignore)",
+ "sha":"(ignore)",
+ "date":0
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json
new file mode 100644
index 00000000000..e293d85b594
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json
@@ -0,0 +1,4 @@
+{
+ "error-code": "BAD_REQUEST",
+ "message": "Got notified about completion of job status of productionUsEast3[ last triggered: (never), last completed: (never), first failing: (not failing), lastSuccess: (never)], but that has not been triggered nor deployed"
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java
new file mode 100644
index 00000000000..5a89ebb2d3f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/MockRoutingGenerator.java
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author bratseth
+ */
+public class MockRoutingGenerator implements RoutingGenerator {
+
+ @Override
+ public List<RoutingEndpoint> endpoints(DeploymentId deployment) {
+ List<RoutingEndpoint> endpoints = new ArrayList<>();
+ endpoints.add(new RoutingEndpoint("qrs-endpoint", false));
+ endpoints.add(new RoutingEndpoint("feeding-endpoint", false));
+ endpoints.add(new RoutingEndpoint("global-endpoint", true));
+ endpoints.add(new RoutingEndpoint("alias-endpoint", true));
+ return endpoints;
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
new file mode 100644
index 00000000000..11e55edb5a5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -0,0 +1,283 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import org.junit.Test;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
+import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test computing of version status
+ *
+ * @author bratseth
+ */
+public class VersionStatusTest {
+
+ @Test
+ public void testEmptyVersionStatus() {
+ VersionStatus status = VersionStatus.empty();
+ assertFalse(status.systemVersion().isPresent());
+ assertTrue(status.versions().isEmpty());
+ }
+
+ @Test
+ public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() {
+ ControllerTester tester = new ControllerTester();
+ Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1);
+ tester.configServerClientMock().setDefaultConfigServerVersion(largerThanCurrent);
+ VersionStatus versionStatus = VersionStatus.compute(tester.controller());
+ assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber());
+ }
+
+ @Test
+ public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException {
+ ControllerTester tester = new ControllerTester();
+ Version oldest = new Version(5);
+ tester.configServerClientMock().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest);
+ VersionStatus versionStatus = VersionStatus.compute(tester.controller());
+ assertEquals(oldest, versionStatus.systemVersion().get().versionNumber());
+ }
+
+ @Test
+ public void testVersionStatusAfterApplicationUpdates() {
+ DeploymentTester tester = new DeploymentTester();
+ ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("default")
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .region("us-east-3")
+ .build();
+
+ // Application versions which are older than the current version
+ Version version1 = new Version("5.1");
+ Version version2 = new Version("5.2");
+ tester.upgradeSystem(version1);
+
+ // Setup applications
+ Application app1 = tester.createAndDeploy("app1", 11, applicationPackage);
+ Application app2 = tester.createAndDeploy("app2", 22, applicationPackage);
+ Application app3 = tester.createAndDeploy("app3", 33, applicationPackage);
+
+ // version2 is released
+ tester.upgradeSystem(version2);
+
+ // - app1 is in production on version1, but then fails in system test on version2
+ tester.completeUpgradeWithError(app1, version2, applicationPackage, systemTest);
+ // - app2 is partially in production on version1 and partially on version2
+ tester.completeUpgradeWithError(app2, version2, applicationPackage, productionUsEast3);
+ // - app3 is in production on version1, but then fails in staging test on version2
+ tester.completeUpgradeWithError(app3, version2, applicationPackage, stagingTest);
+
+ VersionStatus versionStatus = VersionStatus.compute(tester.controller());
+ List<VespaVersion> versions = versionStatus.versions();
+ assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size());
+
+ VespaVersion v0 = versions.get(2);
+ assertEquals(tester.configServerClientMock().getDefaultConfigServerVersion(), v0.versionNumber());
+ assertEquals(0, v0.statistics().failing().size());
+ assertEquals(0, v0.statistics().production().size());
+
+ VespaVersion v1 = versions.get(0);
+ assertEquals(version1, v1.versionNumber());
+ assertEquals(0, v1.statistics().failing().size());
+ // All applications are on v1 in at least one zone
+ assertEquals(3, v1.statistics().production().size());
+ assertTrue(v1.statistics().production().contains(app2.id()));
+ assertTrue(v1.statistics().production().contains(app1.id()));
+
+ VespaVersion v2 = versions.get(1);
+ assertEquals(version2, v2.versionNumber());
+ // All applications have failed on v2 in at least one zone
+ assertEquals(3, v2.statistics().failing().size());
+ assertTrue(v2.statistics().failing().contains(app1.id()));
+ assertTrue(v2.statistics().failing().contains(app3.id()));
+ // Only one application is on v2 in at least one zone
+ assertEquals(1, v2.statistics().production().size());
+ assertTrue(v2.statistics().production().contains(app2.id()));
+
+ VespaVersion v3 = versions.get(3);
+ assertEquals(Vtag.currentVersion, v3.versionNumber());
+ assertEquals(0, v3.statistics().failing().size());
+ assertEquals(0, v3.statistics().production().size());
+ }
+
+ @Test
+ public void testVersionConfidence() {
+ DeploymentTester tester = new DeploymentTester();
+
+ Version version0 = new Version("5.0");
+ tester.upgradeSystem(version0);
+
+ // Setup applications
+ Application canary0 = tester.createAndDeploy("canary0", 0, "canary");
+ Application canary1 = tester.createAndDeploy("canary1", 1, "canary");
+ Application canary2 = tester.createAndDeploy("canary2", 2, "canary");
+ Application default0 = tester.createAndDeploy("default0", 00, "default");
+ Application default1 = tester.createAndDeploy("default1", 11, "default");
+ Application default2 = tester.createAndDeploy("default2", 22, "default");
+ Application default3 = tester.createAndDeploy("default3", 33, "default");
+ Application default4 = tester.createAndDeploy("default4", 44, "default");
+ Application default5 = tester.createAndDeploy("default5", 55, "default");
+ Application default6 = tester.createAndDeploy("default6", 66, "default");
+ Application default7 = tester.createAndDeploy("default7", 77, "default");
+ Application default8 = tester.createAndDeploy("default8", 88, "default");
+ Application default9 = tester.createAndDeploy("default9", 99, "default");
+ Application conservative0 = tester.createAndDeploy("conservative1", 000, "conservative");
+
+
+ // The following applications should not affect confidence calculation:
+
+ // Application without deployment
+ Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L);
+
+ // Pull request build
+ Application ignored1 = tester.controllerTester().createApplication(new TenantId("tenant1"),
+ "ignored1",
+ "default-pr42", 1000);
+
+ Version version1 = new Version("5.1");
+ Version version2 = new Version("5.2");
+ tester.upgradeSystem(version1);
+
+ // Canaries upgrade to new versions and fail
+ tester.completeUpgrade(canary0, version1, "canary");
+ tester.completeUpgradeWithError(canary1, version1, "canary", productionUsWest1);
+ tester.upgradeSystem(version2);
+ tester.completeUpgrade(canary2, version2, "canary");
+
+ VersionStatus versionStatus = VersionStatus.compute(tester.controller());
+ List<VespaVersion> versions = versionStatus.versions();
+
+ assertEquals("One canary failed: Broken",
+ VespaVersion.Confidence.broken, confidence(versions, version1));
+ assertEquals("Nothing has failed but not all canaries has deployed: Low",
+ VespaVersion.Confidence.low, confidence(versions, version2));
+ assertEquals("Current version of this - no deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+
+ // All canaries are upgraded to version2 which raises confidence to normal and more apps upgrade
+ tester.completeUpgrade(canary0, version2, "canary");
+ tester.completeUpgrade(canary1, version2, "canary");
+ tester.upgradeSystem(version2);
+ tester.completeUpgrade(default0, version2, "default");
+ tester.completeUpgrade(default1, version2, "default");
+ tester.completeUpgrade(default2, version2, "default");
+ tester.completeUpgrade(default3, version2, "default");
+ tester.completeUpgrade(default4, version2, "default");
+ tester.completeUpgrade(default5, version2, "default");
+ tester.completeUpgrade(default6, version2, "default");
+ tester.completeUpgrade(default7, version2, "default");
+
+ versionStatus = VersionStatus.compute(tester.controller());
+ versions = versionStatus.versions();
+
+ assertEquals("No deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, version0));
+ assertEquals("All canaries deployed + < 90% of defaults: Normal",
+ VespaVersion.Confidence.normal, confidence(versions, version2));
+ assertEquals("Current version of this - no deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+
+ // Another default application upgrades, raising confidence to high
+ tester.completeUpgrade(default8, version2, "default");
+
+ versionStatus = VersionStatus.compute(tester.controller());
+ versions = versionStatus.versions();
+
+ assertEquals("No deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, version0));
+ assertEquals("90% of defaults deployed successfully: High",
+ VespaVersion.Confidence.high, confidence(versions, version2));
+ assertEquals("Current version of this - no deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+
+ // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version
+ // as broken
+ Version version3 = new Version("5.3");
+ tester.upgradeSystem(version3);
+ tester.completeUpgrade(canary0, version3, "canary");
+ tester.completeUpgrade(canary1, version3, "canary");
+ tester.completeUpgrade(canary2, version3, "canary");
+ tester.upgradeSystem(version3);
+ tester.completeUpgradeWithError(default0, version3, "default", stagingTest);
+ tester.completeUpgradeWithError(default1, version3, "default", stagingTest);
+ tester.completeUpgradeWithError(default2, version3, "default", stagingTest);
+ tester.completeUpgradeWithError(default9, version3, "default", stagingTest);
+
+ versionStatus = VersionStatus.compute(tester.controller());
+ versions = versionStatus.versions();
+
+ assertEquals("No deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, version0));
+ assertEquals("40% of defaults failed: Broken",
+ VespaVersion.Confidence.broken, confidence(versions, version3));
+ assertEquals("Current version of this - no deployments: Low",
+ VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+ }
+
+ @Test
+ public void testComputeIgnoresVersionWithUnknownGitMetadata() {
+ ControllerTester tester = new ControllerTester();
+ ApplicationController applications = tester.controller().applications();
+
+ tester.gitHubClientMock()
+ .mockAny(false)
+ .knownTag(Vtag.currentVersion.toFullString(), "foo") // controller
+ .knownTag("6.1.0", "bar"); // config server
+
+ Version versionWithUnknownTag = new Version("6.1.2");
+
+ Application app = tester.createAndDeploy("tenant1", "domain1","application1", Environment.test, 11);
+ applications.notifyJobCompletion(mockReport(app, component, true));
+ applications.notifyJobCompletion(mockReport(app, systemTest, true));
+
+ List<VespaVersion> vespaVersions = VersionStatus.compute(tester.controller()).versions();
+
+ assertEquals(2, vespaVersions.size()); // controller and config server
+ assertTrue("Version referencing unknown tag is skipped",
+ vespaVersions.stream().noneMatch(v -> v.versionNumber().equals(versionWithUnknownTag)));
+ }
+
+ private VespaVersion.Confidence confidence(List<VespaVersion> versions, Version version) {
+ return versions.stream()
+ .filter(v -> v.statistics().version().equals(version))
+ .findFirst()
+ .map(VespaVersion::confidence)
+ .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version));
+ }
+
+ private DeploymentJobs.JobReport mockReport(Application application, DeploymentJobs.JobType jobType, boolean success) {
+ return new DeploymentJobs.JobReport(
+ application.id(),
+ jobType,
+ application.deploymentJobs().projectId().get(),
+ 1L,
+ JobError.from(success),
+ false,
+ true
+ );
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java
new file mode 100644
index 00000000000..4e2e4bb15b4
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/restapi/impl/StatusPageResourceTest.java
@@ -0,0 +1,65 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.restapi.impl;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.UriBuilder;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author frodelu
+ */
+public class StatusPageResourceTest {
+
+ private StatusPageResource statusPage;
+
+ @Before
+ public void setup() throws IOException {
+
+ Client mockClient = Mockito.mock(Client.class);
+ WebTarget mockTarget = Mockito.mock(WebTarget.class);
+ Invocation.Builder mockRequest = Mockito.mock(Invocation.Builder.class);
+ KeyService keyService = Mockito.mock(KeyService.class);
+
+ Mockito.when(mockClient.target(Mockito.any(UriBuilder.class))).thenReturn(mockTarget);
+ Mockito.when(mockTarget.request()).thenReturn(mockRequest);
+ Mockito.when(mockRequest.get(JsonNode.class)).thenReturn(
+ new ObjectMapper().readTree("{\"page\":{\"name\":\"Vespa\"}}"));
+ Mockito.when(keyService.getSecret(Mockito.any(String.class))).thenReturn("testpage:testkey");
+
+ statusPage = new StatusPageResource(keyService, mockClient);
+ }
+
+
+ @Test
+ public void default_url() {
+ UriBuilder uri = statusPage.statusPageURL("incidents", null);
+ assertNotNull("URI not initialized", uri);
+ assertEquals("https://testpage.statuspage.io/api/v2/incidents.json?api_key=testkey", uri.toTemplate());
+ }
+
+ @Test
+ public void url_with_since_param() {
+ UriBuilder uri = statusPage.statusPageURL("incidents", "2015-01-01T00:00+00:00");
+ assertNotNull("URI not initialized", uri);
+ assertEquals("https://testpage.statuspage.io/api/v2/incidents.json?api_key=testkey&since=2015-01-01T00%3A00%2B00%3A00", uri.toTemplate());
+ }
+
+ @Test
+ public void valid_status_page() {
+ JsonNode result = statusPage.statusPage("incidents", null);
+ assertNotNull("No result from StatusPage.io", result);
+ assertEquals("Vespa", result.get("page").get("name").asText());
+ }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java
new file mode 100644
index 00000000000..8fc218a9e8b
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java
@@ -0,0 +1,200 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.rotation;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId;
+import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
+import com.yahoo.vespa.hosted.controller.persistence.ControllerDb;
+import com.yahoo.vespa.hosted.controller.persistence.MemoryControllerDb;
+import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.StringReader;
+import java.net.URI;
+import java.util.Collections;
+import java.util.Set;
+
+import static org.fest.assertions.Assertions.*;
+
+/**
+ * @author Oyvind Gronnesby
+ */
+public class ControllerRotationRepositoryTest {
+
+ private final RotationsConfig rotationsConfig = new RotationsConfig(
+ new RotationsConfig.Builder()
+ .rotations("foo-1", "foo-1.com")
+ .rotations("foo-2", "foo-2.com")
+ );
+ private final RotationsConfig rotationsConfigWhitespaces = new RotationsConfig(
+ new RotationsConfig.Builder()
+ .rotations("foo-1", "\n foo-1.com \n")
+ .rotations("foo-2", "foo-2.com")
+ );
+ private final ControllerDb controllerDb = new MemoryControllerDb();
+ private final ApplicationId applicationId = ApplicationId.from("msbe", "tumblr-search", "default");
+
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ private final DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(
+ new StringReader(
+ "<deployment>" +
+ " <prod global-service-id='foo'>" +
+ " <region active='true'>us-east</region>" +
+ " <region active='true'>us-west</region>" +
+ " </prod>" +
+ "</deployment>"
+ )
+ );
+
+ private final DeploymentSpec deploymentSpecOneRegion = DeploymentSpec.fromXml(
+ new StringReader(
+ "<deployment>" +
+ " <prod global-service-id='nalle'>" +
+ " <region active='true'>us-east</region>" +
+ " </prod>" +
+ "</deployment>"
+ )
+ );
+
+ private final DeploymentSpec deploymentSpecNoServiceId = DeploymentSpec.fromXml(
+ new StringReader(
+ "<deployment>" +
+ " <prod>" +
+ " <region active='true'>us-east</region>" +
+ " <region active='true'>us-west</region>" +
+ " </prod>" +
+ "</deployment>"
+ )
+ );
+
+ private final DeploymentSpec deploymentSpecOnlyOneNonCorpRegion = DeploymentSpec.fromXml(
+ new StringReader(
+ "<deployment>" +
+ " <prod global-service-id='nalle'>" +
+ " <region active='true'>us-east</region>" +
+ " <region active='true'>corp-us-west</region>" +
+ " </prod>" +
+ "</deployment>"
+ )
+ );
+
+ private final DeploymentSpec deploymentSpecWithAdditionalCorpZone = DeploymentSpec.fromXml(
+ new StringReader(
+ "<deployment>" +
+ " <prod global-service-id='nalle'>" +
+ " <region active='true'>us-east</region>" +
+ " <region active='true'>corp-us-west</region>" +
+ " <region active='true'>us-west</region>" +
+ " </prod>" +
+ "</deployment>"
+ )
+ );
+
+ private ControllerRotationRepository repository;
+ private ControllerRotationRepository repositoryWhitespaces;
+
+
+ @Before
+ public void setup_repository() {
+ repository = new ControllerRotationRepository(rotationsConfig, controllerDb, MetricReceiver.nullImplementation);
+ repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, MetricReceiver.nullImplementation);
+ controllerDb.assignRotation(new RotationId("foo-1"), applicationId);
+ }
+
+ @Test
+ public void application_with_rotation_reused() {
+ Set<Rotation> rotations = repository.getOrAssignRotation(applicationId, deploymentSpec);
+ Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com");
+ assertThat(rotations).containsOnly(assignedRotation);
+ }
+
+ @Test
+ public void names_stripped() {
+ Set<Rotation> rotations = repositoryWhitespaces.getOrAssignRotation(applicationId, deploymentSpec);
+ Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com");
+ assertThat(rotations).containsOnly(assignedRotation);
+ }
+
+ @Test
+ public void application_without_rotation() {
+ ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default");
+ Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpec);
+ Rotation assignedRotation = new Rotation(new RotationId("foo-2"), "foo-2.com");
+ assertThat(rotations).containsOnly(assignedRotation);
+ }
+
+ @Test
+ public void application_without_rotation_but_none_left() {
+ application_without_rotation(); // run this test to assign last rotation
+ ApplicationId third = ApplicationId.from("thirdtenant", "thirdapplication", "default");
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("no rotations available");
+
+ repository.getOrAssignRotation(third, deploymentSpec);
+ }
+
+ @Test
+ public void application_without_rotation_but_does_not_qualify() {
+ ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default");
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("less than 2 prod zones are defined");
+
+ repository.getOrAssignRotation(other, deploymentSpecOneRegion);
+ }
+
+ @Test
+ public void application_with_rotation_but_does_not_qualify() {
+ Set<Rotation> rotations = repository.getOrAssignRotation(applicationId, deploymentSpecOneRegion);
+ Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com");
+ assertThat(rotations).containsOnly(assignedRotation);
+ }
+
+ @Test
+ public void application_with_rotation_is_listed() {
+ repository.getOrAssignRotation(applicationId, deploymentSpec);
+ Set<URI> uris = repository.getRotationUris(applicationId);
+ assertThat(uris).isEqualTo(
+ Collections.singleton(URI.create("http://tumblr-search.msbe.global.vespa.yahooapis.com:4080/"))
+ );
+ }
+
+ @Test
+ public void application_without_rotation_is_empty() {
+ ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default");
+ Set<URI> uris = repository.getRotationUris(other);
+ assertThat(uris).isEmpty();
+ }
+
+ @Test
+ public void application_without_serviceid_and_two_regions() {
+ ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default");
+ Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpecNoServiceId);
+ assertThat(rotations).isEmpty();
+ }
+
+ @Test
+ public void application_with_only_one_non_corp_region() {
+ ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default");
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("less than 2 prod zones are defined");
+
+ repository.getOrAssignRotation(other, deploymentSpecOnlyOneNonCorpRegion);
+ }
+
+ @Test
+ public void application_with_corp_region_and_two_non_corp_region() {
+ ApplicationId other = ApplicationId.from("othertenant", "otherapplication", "default");
+ Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpecWithAdditionalCorpZone);
+ assertThat(rotations).containsOnly(new Rotation(new RotationId("foo-2"), "foo-2.com"));
+ }
+
+}
diff --git a/controller-server/src/test/resources/chef_output.json b/controller-server/src/test/resources/chef_output.json
new file mode 100644
index 00000000000..257065f7b5b
--- /dev/null
+++ b/controller-server/src/test/resources/chef_output.json
@@ -0,0 +1,34 @@
+{
+ "total": 1,
+ "start": 0,
+ "rows": [
+ {
+ "url": "https://chef-server.test/organizations/vespa/nodes/fake-node.test",
+ "data": {
+ "fqdn": "fake-node.test",
+ "ohai_time": 1475497186.68962,
+ "tenant": "ciintegrationtests",
+ "application": "restart",
+ "instance": "default",
+ "zone": "cd_cd-us-east-1_prod",
+ "system": "cd",
+ "environment": "prod",
+ "region": "cd-us-east-1"
+ }
+ },
+ {
+ "url": "https://chef-server.test/organizations/vespa/nodes/fake-node2.test",
+ "data": {
+ "fqdn": "fake-node2.test",
+ "ohai_time": 1475497186.68962,
+ "tenant": null,
+ "application": null,
+ "instance": null,
+ "zone": null,
+ "system": null,
+ "environment": null,
+ "region": null
+ }
+ }
+ ]
+}
diff --git a/controller-server/src/test/resources/job-grandparent.json b/controller-server/src/test/resources/job-grandparent.json
new file mode 100644
index 00000000000..63602bed146
--- /dev/null
+++ b/controller-server/src/test/resources/job-grandparent.json
@@ -0,0 +1,4 @@
+{
+ "duration": 720000,
+ "causes": []
+}
diff --git a/controller-server/src/test/resources/job-parent.json b/controller-server/src/test/resources/job-parent.json
new file mode 100644
index 00000000000..88d50de394f
--- /dev/null
+++ b/controller-server/src/test/resources/job-parent.json
@@ -0,0 +1,9 @@
+{
+ "duration": 1200000,
+ "causes": [
+ {
+ "upstreamBuild": 231,
+ "upstreamProject": "3-v3-job-grandparent"
+ }
+ ]
+}
diff --git a/controller-server/src/test/resources/job.json b/controller-server/src/test/resources/job.json
new file mode 100644
index 00000000000..845566867b7
--- /dev/null
+++ b/controller-server/src/test/resources/job.json
@@ -0,0 +1,9 @@
+{
+ "duration": 600000,
+ "causes": [
+ {
+ "upstreamBuild": 123,
+ "upstreamProject": "2-v3-job-parent"
+ }
+ ]
+}
diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp
index 7cad7356c8e..d8920b0577b 100644
--- a/documentapi/src/tests/messagebus/messagebus_test.cpp
+++ b/documentapi/src/tests/messagebus/messagebus_test.cpp
@@ -81,7 +81,7 @@ void Test::testMessage() {
new document::DocumentUpdate(*testdoc_type,
document::DocumentId(document::DocIdString(
"testdoc", "testme2")))));
- EXPECT_TRUE(!(upd1.getDocumentUpdate()->getId() == upd2.getDocumentUpdate()->getId()));
+ EXPECT_TRUE(!(upd1.getDocumentUpdate().getId() == upd2.getDocumentUpdate().getId()));
DocumentMessage& msg2 = static_cast<DocumentMessage&>(upd2);
EXPECT_TRUE(msg2.getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT);
diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp
index 8c20ef77201..964e4e12288 100644
--- a/documentapi/src/tests/messages/messages50test.cpp
+++ b/documentapi/src/tests/messages/messages50test.cpp
@@ -444,8 +444,8 @@ Messages50Test::testPutDocumentMessage()
mbus::Routable::UP obj = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang);
if (EXPECT_TRUE(obj.get() != NULL)) {
PutDocumentMessage &ref = static_cast<PutDocumentMessage&>(*obj);
- EXPECT_TRUE(ref.getDocument()->getType().getName() == "testdoc");
- EXPECT_TRUE(ref.getDocument()->getId().toString() == "doc:scheme:");
+ EXPECT_TRUE(ref.getDocument().getType().getName() == "testdoc");
+ EXPECT_TRUE(ref.getDocument().getId().toString() == "doc:scheme:");
EXPECT_EQUAL(666u, ref.getTimestamp());
EXPECT_EQUAL(37u, ref.getApproxSize());
}
@@ -737,7 +737,7 @@ Messages50Test::testUpdateDocumentMessage()
mbus::Routable::UP obj = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang);
if (EXPECT_TRUE(obj.get() != NULL)) {
UpdateDocumentMessage &ref = static_cast<UpdateDocumentMessage&>(*obj);
- EXPECT_EQUAL(*upd, *ref.getDocumentUpdate());
+ EXPECT_EQUAL(*upd, ref.getDocumentUpdate());
EXPECT_EQUAL(666u, ref.getOldTimestamp());
EXPECT_EQUAL(777u, ref.getNewTimestamp());
EXPECT_EQUAL(85u, ref.getApproxSize());
@@ -1047,8 +1047,8 @@ Messages50Test::testGetDocumentReply()
if (EXPECT_TRUE(obj.get() != NULL)) {
GetDocumentReply &ref = static_cast<GetDocumentReply&>(*obj);
- EXPECT_EQUAL(string("testdoc"), ref.getDocument()->getType().getName());
- EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument()->getId().toString());
+ EXPECT_EQUAL(string("testdoc"), ref.getDocument().getType().getName());
+ EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument().getId().toString());
}
}
return true;
diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp
index d6394012688..33eb5134dce 100644
--- a/documentapi/src/tests/messages/messages52test.cpp
+++ b/documentapi/src/tests/messages/messages52test.cpp
@@ -52,8 +52,8 @@ Messages52Test::testPutDocumentMessage()
if (EXPECT_TRUE(routableUp.get() != nullptr)) {
auto & deserializedMsg = static_cast<PutDocumentMessage &>(*routableUp);
- EXPECT_EQUAL(msg.getDocument()->getType().getName(), deserializedMsg.getDocument()->getType().getName());
- EXPECT_EQUAL(msg.getDocument()->getId().toString(), deserializedMsg.getDocument()->getId().toString());
+ EXPECT_EQUAL(msg.getDocument().getType().getName(), deserializedMsg.getDocument().getType().getName());
+ EXPECT_EQUAL(msg.getDocument().getId().toString(), deserializedMsg.getDocument().getId().toString());
EXPECT_EQUAL(msg.getTimestamp(), deserializedMsg.getTimestamp());
EXPECT_EQUAL(67u, deserializedMsg.getApproxSize());
EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
@@ -90,8 +90,7 @@ Messages52Test::testUpdateDocumentMessage()
const DocumentTypeRepo & repo = getTypeRepo();
const document::DocumentType & docType = *repo.getDocumentType("testdoc");
- document::DocumentUpdate::SP docUpdate(new document::DocumentUpdate(docType,
- document::DocumentId("doc:scheme:")));
+ auto docUpdate = std::make_shared<document::DocumentUpdate>(docType, document::DocumentId("doc:scheme:"));
docUpdate->addFieldPathUpdate(document::FieldPathUpdate::CP(
new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0")));
@@ -108,7 +107,7 @@ Messages52Test::testUpdateDocumentMessage()
if (EXPECT_TRUE(routableUp.get() != nullptr)) {
auto & deserializedMsg = static_cast<UpdateDocumentMessage &>(*routableUp);
- EXPECT_EQUAL(*msg.getDocumentUpdate(), *deserializedMsg.getDocumentUpdate());
+ EXPECT_EQUAL(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate());
EXPECT_EQUAL(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp());
EXPECT_EQUAL(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp());
EXPECT_EQUAL(115u, deserializedMsg.getApproxSize());
diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp
index 0eb08f1e632..3629604aeea 100644
--- a/documentapi/src/tests/policies/policies_test.cpp
+++ b/documentapi/src/tests/policies/policies_test.cpp
@@ -738,7 +738,7 @@ Test::multipleGetRepliesAreMergedToFoundDocument()
doc.reset(new Document(*_docType, DocumentId("doc:scheme:yarn")));
doc->setLastModified(123456ULL);
}
- mbus::Reply::UP reply(new GetDocumentReply(doc));
+ mbus::Reply::UP reply(new GetDocumentReply(std::move(doc)));
selected[i]->handleReply(std::move(reply));
}
mbus::Reply::UP reply = frame.getReceptor().getReply(600);
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp
index 41431dc305e..c7422a529a3 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.cpp
@@ -2,6 +2,7 @@
#include "getdocumentreply.h"
#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/document/fieldvalue/document.h>
namespace documentapi {
@@ -15,30 +16,18 @@ GetDocumentReply::~GetDocumentReply() {}
GetDocumentReply::GetDocumentReply(document::Document::SP document) :
DocumentAcceptedReply(DocumentProtocol::REPLY_GETDOCUMENT),
- _document(document),
+ _document(std::move(document)),
_lastModified(0)
{
- if (_document.get()) {
+ if (_document) {
_lastModified = _document->getLastModified();
}
}
-document::Document::SP
-GetDocumentReply::getDocument()
-{
- return _document;
-}
-
-std::shared_ptr<const document::Document>
-GetDocumentReply::getDocument() const
-{
- return _document;
-}
-
void
GetDocumentReply::setDocument(document::Document::SP document)
{
- _document = document;
+ _document = std::move(document);
if (document.get()) {
_lastModified = document->getLastModified();
} else {
@@ -46,10 +35,4 @@ GetDocumentReply::setDocument(document::Document::SP document)
}
}
-void
-GetDocumentReply::setLastModified(uint64_t lastModified)
-{
- _lastModified = lastModified;
-}
-
}
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h
index 04859af51ea..2cff36325ab 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/getdocumentreply.h
@@ -2,14 +2,16 @@
#pragma once
#include "documentacceptedreply.h"
-#include <vespa/document/fieldvalue/document.h>
+
+namespace document { class Document; }
namespace documentapi {
class GetDocumentReply : public DocumentAcceptedReply {
private:
- document::Document::SP _document;
- uint64_t _lastModified;
+ using DocumentSP = std::shared_ptr<document::Document>;
+ DocumentSP _document;
+ uint64_t _lastModified;
public:
/**
@@ -29,28 +31,22 @@ public:
*
* @param document The document requested.
*/
- GetDocumentReply(document::Document::SP document);
-
- /**
- * Returns the document retrieved.
- *
- * @return The document.
- */
- document::Document::SP getDocument();
+ GetDocumentReply(DocumentSP document);
/**
* Returns the document retrieved.
*
* @return The document.
*/
- std::shared_ptr<const document::Document> getDocument() const;
+ const document::Document & getDocument() const { return *_document; }
+ bool hasDocument() const { return _document.get() != nullptr; }
/**
* Sets the document retrieved.
*
* @param document The document.
*/
- void setDocument(document::Document::SP document);
+ void setDocument(DocumentSP document);
/**
* Returns the date the document was last modified.
@@ -64,7 +60,7 @@ public:
*
* @param lastModified The date.
*/
- void setLastModified(uint64_t lastModified);
+ void setLastModified(uint64_t lastModified) { _lastModified = lastModified; }
string toString() const override { return "getdocumentreply"; }
};
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp
index efd2e405267..6753d269ad6 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp
@@ -2,6 +2,7 @@
#include "putdocumentmessage.h"
#include "writedocumentreply.h"
#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/document/fieldvalue/document.h>
#include <vespa/vespalib/util/exceptions.h>
namespace documentapi {
@@ -17,7 +18,7 @@ PutDocumentMessage::PutDocumentMessage(document::Document::SP document) :
_document(),
_time(0)
{
- setDocument(document);
+ setDocument(std::move(document));
}
PutDocumentMessage::~PutDocumentMessage() {}
@@ -46,25 +47,13 @@ PutDocumentMessage::getType() const
return DocumentProtocol::MESSAGE_PUTDOCUMENT;
}
-document::Document::SP
-PutDocumentMessage::getDocument()
-{
- return _document;
-}
-
-std::shared_ptr<const document::Document>
-PutDocumentMessage::getDocument() const
-{
- return _document;
-}
-
void
PutDocumentMessage::setDocument(document::Document::SP document)
{
- if (document.get() == NULL) {
+ if ( ! document ) {
throw vespalib::IllegalArgumentException("Document can not be null.", VESPA_STRLOC);
}
- _document = document;
+ _document = std::move(document);
}
}
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h
index e4b43aaaf37..9bc1c088dfa 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h
@@ -2,25 +2,22 @@
#pragma once
#include "testandsetmessage.h"
-#include <vespa/document/fieldvalue/document.h>
+namespace document { class Document; }
namespace documentapi {
class PutDocumentMessage : public TestAndSetMessage {
private:
- document::Document::SP _document;
- uint64_t _time;
+ using DocumentSP = std::shared_ptr<document::Document>;
+ DocumentSP _document;
+ uint64_t _time;
protected:
- // Implements DocumentMessage.
DocumentReply::UP doCreateReply() const override;
public:
- /**
- * Convenience typedef.
- */
- typedef std::unique_ptr<PutDocumentMessage> UP;
- typedef std::shared_ptr<PutDocumentMessage> SP;
+ using UP = std::unique_ptr<PutDocumentMessage>;
+ using SP = std::shared_ptr<PutDocumentMessage>;
/**
* Constructs a new document message for deserialization.
@@ -32,7 +29,7 @@ public:
*
* @param document The document to put.
*/
- PutDocumentMessage(document::Document::SP document);
+ PutDocumentMessage(DocumentSP document);
~PutDocumentMessage();
/**
@@ -40,21 +37,15 @@ public:
*
* @return The document.
*/
- document::Document::SP getDocument();
-
- /**
- * Returns the document to put.
- *
- * @return The document.
- */
- std::shared_ptr<const document::Document> getDocument() const;
+ const DocumentSP & getDocumentSP() const { return _document; }
+ const document::Document & getDocument() const { return *_document; }
/**
* Sets the document to put.
*
* @param document The document to set.
*/
- void setDocument(document::Document::SP document);
+ void setDocument(DocumentSP document);
/**
* Returns the timestamp of the document to put.
@@ -69,13 +60,9 @@ public:
* @param time The timestamp to set.
*/
void setTimestamp(uint64_t time) { _time = time; }
-
bool hasSequenceId() const override;
-
uint64_t getSequenceId() const override;
-
uint32_t getType() const override;
-
string toString() const override { return "putdocumentmessage"; }
};
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp
index ef8a2b74298..db5dafce271 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.cpp
@@ -3,6 +3,7 @@
#include "updatedocumentmessage.h"
#include "updatedocumentreply.h"
#include <vespa/documentapi/messagebus/documentprotocol.h>
+#include <vespa/document/update/documentupdate.h>
#include <vespa/vespalib/util/exceptions.h>
namespace documentapi {
@@ -20,7 +21,7 @@ UpdateDocumentMessage::UpdateDocumentMessage(document::DocumentUpdate::SP docume
_oldTime(0),
_newTime(0)
{
- setDocumentUpdate(documentUpdate);
+ setDocumentUpdate(std::move(documentUpdate));
}
UpdateDocumentMessage::~UpdateDocumentMessage() {}
@@ -49,25 +50,13 @@ UpdateDocumentMessage::getType() const
return DocumentProtocol::MESSAGE_UPDATEDOCUMENT;
}
-document::DocumentUpdate::SP
-UpdateDocumentMessage::getDocumentUpdate()
-{
- return _documentUpdate;
-}
-
-std::shared_ptr<const document::DocumentUpdate>
-UpdateDocumentMessage::getDocumentUpdate() const
-{
- return _documentUpdate;
-}
-
void
UpdateDocumentMessage::setDocumentUpdate(document::DocumentUpdate::SP documentUpdate)
{
- if (documentUpdate.get() == NULL) {
+ if ( ! documentUpdate) {
throw vespalib::IllegalArgumentException("Document update can not be null.", VESPA_STRLOC);
}
- _documentUpdate = documentUpdate;
+ _documentUpdate = std::move(documentUpdate);
}
}
diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h
index 25991191eeb..3a320960515 100644
--- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h
+++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h
@@ -2,15 +2,17 @@
#pragma once
#include "testandsetmessage.h"
-#include <vespa/document/update/documentupdate.h>
+
+namespace document { class DocumentUpdate; }
namespace documentapi {
class UpdateDocumentMessage : public TestAndSetMessage {
private:
- document::DocumentUpdate::SP _documentUpdate;
- uint64_t _oldTime;
- uint64_t _newTime;
+ using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>;
+ DocumentUpdateSP _documentUpdate;
+ uint64_t _oldTime;
+ uint64_t _newTime;
protected:
DocumentReply::UP doCreateReply() const override;
@@ -33,28 +35,21 @@ public:
*
* @param documentUpdate The document update to perform.
*/
- UpdateDocumentMessage(document::DocumentUpdate::SP documentUpdate);
-
- /**
- * Returns the document update to perform.
- *
- * @return The update.
- */
- document::DocumentUpdate::SP getDocumentUpdate();
+ UpdateDocumentMessage(DocumentUpdateSP documentUpdate);
/**
* Returns the document update to perform.
*
* @return The update.
*/
- std::shared_ptr<const document::DocumentUpdate> getDocumentUpdate() const;
-
+ const DocumentUpdateSP & getDocumentUpdateSP() const { return _documentUpdate; }
+ const document::DocumentUpdate & getDocumentUpdate() const { return *_documentUpdate; }
/**
* Sets the document update to perform.
*
* @param documentUpdate The document update to set.
*/
- void setDocumentUpdate(document::DocumentUpdate::SP documentUpdate);
+ void setDocumentUpdate(DocumentUpdateSP documentUpdate);
/**
* Returns the timestamp required for this update to be applied.
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
index 011b54305bb..6756f694267 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp
@@ -124,12 +124,10 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespali
const mbus::Message &msg = context.getMessage();
switch(msg.getType()) {
case DocumentProtocol::MESSAGE_PUTDOCUMENT:
- return it->second->contains(*static_cast<const PutDocumentMessage&>(msg).getDocument()) ==
- Result::True;
+ return it->second->contains(static_cast<const PutDocumentMessage&>(msg).getDocument()) == Result::True;
case DocumentProtocol::MESSAGE_UPDATEDOCUMENT:
- return it->second->contains(*static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) !=
- Result::False;
+ return it->second->contains(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()) != Result::False;
case DocumentProtocol::MESSAGE_MULTIOPERATION:
{
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp
index a6b1e200cd4..38610aca551 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/searchcolumnpolicy.cpp
@@ -53,20 +53,20 @@ SearchColumnPolicy::select(mbus::RoutingContext &context)
const mbus::Message &msg = context.getMessage();
switch(msg.getType()) {
case DocumentProtocol::MESSAGE_PUTDOCUMENT:
- id = &static_cast<const PutDocumentMessage&>(msg).getDocument()->getId();
+ id = &static_cast<const PutDocumentMessage&>(msg).getDocument().getId();
break;
case DocumentProtocol::MESSAGE_GETDOCUMENT:
- id = &static_cast<const GetDocumentMessage&>(msg).getDocumentId();
+ id = &static_cast<const GetDocumentMessage&>(msg).getDocumentId();
break;
case DocumentProtocol::MESSAGE_REMOVEDOCUMENT:
- id = &static_cast<const RemoveDocumentMessage&>(msg).getDocumentId();
+ id = &static_cast<const RemoveDocumentMessage&>(msg).getDocumentId();
break;
case DocumentProtocol::MESSAGE_UPDATEDOCUMENT:
- id = &static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()->getId();
- break;
+ id = &static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate().getId();
+ break;
case DocumentProtocol::MESSAGE_MULTIOPERATION:
bucketId = (static_cast<const MultiOperationMessage&>(msg)).getBucketId();
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
index cace5b6576a..b7b451e8ddf 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp
@@ -119,7 +119,7 @@ StoragePolicy::doSelect(mbus::RoutingContext &context)
document::BucketId id;
switch(msg.getType()) {
case DocumentProtocol::MESSAGE_PUTDOCUMENT:
- id = _bucketIdFactory.getBucketId(static_cast<const PutDocumentMessage&>(msg).getDocument()->getId());
+ id = _bucketIdFactory.getBucketId(static_cast<const PutDocumentMessage&>(msg).getDocument().getId());
break;
case DocumentProtocol::MESSAGE_GETDOCUMENT:
@@ -131,7 +131,7 @@ StoragePolicy::doSelect(mbus::RoutingContext &context)
break;
case DocumentProtocol::MESSAGE_UPDATEDOCUMENT:
- id = _bucketIdFactory.getBucketId(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate()->getId());
+ id = _bucketIdFactory.getBucketId(static_cast<const UpdateDocumentMessage&>(msg).getDocumentUpdate().getId());
break;
case DocumentProtocol::MESSAGE_MULTIOPERATION:
diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
index e1b6035d5e0..504b3fcdf4f 100644
--- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp
@@ -567,14 +567,15 @@ RoutableFactories50::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf
GetDocumentReply &reply = static_cast<GetDocumentReply&>(*ret);
bool hasDocument = decodeBoolean(buf);
- document::Document::SP document;
+ document::Document * document = nullptr;
if (hasDocument) {
- document.reset(new document::Document(_repo, buf));
- reply.setDocument(document);
+ auto doc = std::make_shared<document::Document>(_repo, buf);
+ document = doc.get();
+ reply.setDocument(std::move(doc));
}
int64_t lastModified = decodeLong(buf);
reply.setLastModified(lastModified);
- if (document.get()) {
+ if (hasDocument) {
document->setLastModified(lastModified);
}
@@ -586,10 +587,10 @@ RoutableFactories50::GetDocumentReplyFactory::doEncode(const DocumentReply &obj,
{
const GetDocumentReply &reply = static_cast<const GetDocumentReply&>(obj);
- buf.putByte(reply.getDocument().get() == NULL ? 0 : 1);
- if (reply.getDocument().get() != NULL) {
+ buf.putByte(reply.hasDocument() ? 1 : 0);
+ if (reply.hasDocument()) {
nbostream stream;
- reply.getDocument()->serialize(stream);
+ reply.getDocument().serialize(stream);
buf.putBytes(stream.peek(), stream.size());
}
buf.putLong(reply.getLastModified());
@@ -693,7 +694,7 @@ RoutableFactories50::PutDocumentMessageFactory::doEncode(const DocumentMessage &
auto & msg = static_cast<const PutDocumentMessage &>(obj);
nbostream stream;
- msg.getDocument()->serialize(stream);
+ msg.getDocument().serialize(stream);
buf.putBytes(stream.peek(), stream.size());
buf.putLong(static_cast<int64_t>(msg.getTimestamp()));
@@ -950,7 +951,7 @@ RoutableFactories50::UpdateDocumentMessageFactory::doEncode(const DocumentMessag
const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj);
vespalib::nbostream stream;
- msg.getDocumentUpdate()->serializeHEAD(stream);
+ msg.getDocumentUpdate().serializeHEAD(stream);
buf.putBytes(stream.peek(), stream.size());
buf.putLong((int64_t)msg.getOldTimestamp());
buf.putLong((int64_t)msg.getNewTimestamp());
diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java
index 51b6d285819..921c10a1821 100644
--- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java
+++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogServiceIntegrationTest.java
@@ -8,7 +8,9 @@ import org.osgi.framework.ServiceReference;
import org.osgi.service.log.LogEntry;
import org.osgi.service.log.LogReaderService;
-import java.util.Enumeration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -20,6 +22,7 @@ import static org.junit.Assert.assertTrue;
/**
* @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ * @author bjorncs
*/
public class OsgiLogServiceIntegrationTest {
@@ -35,28 +38,26 @@ public class OsgiLogServiceIntegrationTest {
BundleContext ctx = driver.osgiFramework().bundleContext();
ServiceReference<?> ref = ctx.getServiceReference(LogReaderService.class.getName());
LogReaderService reader = (LogReaderService)ctx.getService(ref);
- Enumeration<LogEntry> log = (Enumeration<LogEntry>)reader.getLog();
+ ArrayList<LogEntry> logEntries = Collections.list(reader.getLog());
+ assertTrue(logEntries.size() >= 4);
- assertEntry(Level.INFO, "[jdk14] hello world", null, now, log);
- assertEntry(Level.INFO, "[slf4j] hello world", null, now, log);
- assertEntry(Level.INFO, "[log4j] hello world", null, now, log);
- assertEntry(Level.INFO, "[jcl] hello world", null, now, log);
+ assertLogContainsEntry("[jdk14] hello world", logEntries, now);
+ assertLogContainsEntry("[slf4j] hello world", logEntries, now);
+ assertLogContainsEntry("[log4j] hello world", logEntries, now);
+ assertLogContainsEntry("[jcl] hello world", logEntries, now);
assertTrue(driver.close());
}
- private static void assertEntry(Level expectedLevel, String expectedMessage, Throwable expectedException,
- long expectedTimeGE, Enumeration<LogEntry> log)
+ private static void assertLogContainsEntry(String expectedMessage, List<LogEntry> logEntries, long expectedTimeGE)
{
- assertTrue(log.hasMoreElements());
- LogEntry entry = log.nextElement();
- assertNotNull(entry);
- System.err.println("log entry: "+entry.getMessage()+" bundle="+entry.getBundle());
- assertEquals(expectedMessage, entry.getMessage());
+ LogEntry entry = logEntries.stream().filter(e -> e.getMessage().equals(expectedMessage)).findFirst()
+ .orElseThrow(() -> new AssertionError("Could not find log entry with messsage: " + expectedMessage));
+
assertNull(entry.getBundle());
assertNotNull(entry.getServiceReference());
- assertEquals(OsgiLogHandler.toServiceLevel(expectedLevel), entry.getLevel());
- assertEquals(expectedException, entry.getException());
+ assertEquals(OsgiLogHandler.toServiceLevel(Level.INFO), entry.getLevel());
+ assertNull(entry.getException());
assertTrue(expectedTimeGE <= entry.getTime());
}
}
diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
index 20edbd084e5..75c5fc3f6c5 100644
--- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
@@ -395,4 +395,3 @@ RPCNetwork::getMirror() const
}
} // namespace mbus
-
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
index c8f64a65415..8f7317b28eb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
@@ -112,7 +112,7 @@ public class History {
public enum Type {
// State move events
- readied, reserved, activated, deactivated, deallocated, parked,
+ provisioned(false), readied, reserved, activated, deactivated, deallocated, parked,
// The active node was retired
retired,
// The active node went down according to the service monitor
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index 8056bd787db..9393dc5ead4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -257,6 +257,7 @@ public class NodeSerializer {
/** Returns the event type, or null if this event type should be ignored */
private History.Event.Type eventTypeFromString(String eventTypeString) {
switch (eventTypeString) {
+ case "provisioned" : return History.Event.Type.provisioned;
case "readied" : return History.Event.Type.readied;
case "reserved" : return History.Event.Type.reserved;
case "activated" : return History.Event.Type.activated;
@@ -273,6 +274,7 @@ public class NodeSerializer {
}
private String toString(History.Event.Type nodeEventType) {
switch (nodeEventType) {
+ case provisioned : return "provisioned";
case readied : return "readied";
case reserved : return "reserved";
case activated : return "activated";
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 1b19b57317e..f91b4863eeb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -117,6 +117,10 @@ public class SerializationTest {
" \"flavor\" : \"large\",\n" +
" \"history\" : [\n" +
" {\n" +
+ " \"type\" : \"provisioned\",\n" +
+ " \"at\" : 1444391401389\n" +
+ " },\n" +
+ " {\n" +
" \"type\" : \"reserved\",\n" +
" \"at\" : 1444391402611\n" +
" }\n" +
@@ -143,6 +147,8 @@ public class SerializationTest {
assertEquals(2, node.status().reboot().current());
assertEquals(3, node.allocation().get().restartGeneration().wanted());
assertEquals(4, node.allocation().get().restartGeneration().current());
+ assertEquals(Arrays.asList(History.Event.Type.provisioned, History.Event.Type.reserved),
+ node.history().events().stream().map(History.Event::type).collect(Collectors.toList()));
assertTrue(node.allocation().get().isRemovable());
assertEquals(NodeType.tenant, node.type());
}
diff --git a/pom.xml b/pom.xml
index c668f1d925d..dd7042bf911 100644
--- a/pom.xml
+++ b/pom.xml
@@ -941,6 +941,8 @@
<module>container-search-and-docproc</module>
<module>container-search</module>
<module>container-test-jars</module>
+ <module>controller-api</module>
+ <module>controller-server</module>
<module>defaults</module>
<module>docker-api</module>
<module>docproc</module>
diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp
index c38811e1962..d942049192a 100644
--- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp
+++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp
@@ -4,13 +4,9 @@
#include <vespa/config/print/fileconfigwriter.h>
#include <vespa/document/config/config-documenttypes.h>
#include <vespa/document/document.h>
-#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/documentapi/loadtypes/loadtypeset.h>
#include <vespa/messagebus/destinationsession.h>
-#include <vespa/messagebus/imessagehandler.h>
-#include <vespa/messagebus/iprotocol.h>
-#include <vespa/messagebus/message.h>
#include <vespa/messagebus/protocolset.h>
#include <vespa/messagebus/rpcmessagebus.h>
#include <vespa/vespalib/io/fileutil.h>
@@ -56,7 +52,7 @@ private:
OutputFile &_dat;
size_t _numDocs;
- void handleDocumentPut(document::Document::SP doc);
+ void handleDocumentPut(const document::Document::SP & doc);
virtual void handleMessage(mbus::Message::UP message) override;
public:
@@ -66,9 +62,9 @@ public:
};
void
-FeedHandler::handleDocumentPut(document::Document::SP doc)
+FeedHandler::handleDocumentPut(const document::Document::SP & doc)
{
- if (doc.get() != 0) {
+ if (doc) {
vespalib::nbostream datStream(12345);
vespalib::nbostream idxStream(12);
doc->serialize(datStream);
@@ -86,7 +82,7 @@ FeedHandler::handleMessage(mbus::Message::UP message)
documentapi::DocumentMessage::UP msg((documentapi::DocumentMessage*)message.release());
switch (msg->getType()) {
case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT:
- handleDocumentPut(((documentapi::PutDocumentMessage&)(*msg)).getDocument());
+ handleDocumentPut(((documentapi::PutDocumentMessage&)(*msg)).getDocumentSP());
break;
default:
break;
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index 4083a7e1194..f0a91a101eb 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -20,13 +20,13 @@
#include <vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h>
#include <vespa/searchcore/proton/server/summaryadapter.h>
#include <vespa/searchcore/proton/server/reconfig_params.h>
+#include <vespa/searchcore/proton/matching/sessionmanager.h>
#include <vespa/searchcore/proton/test/documentdb_config_builder.h>
#include <vespa/searchcore/proton/test/mock_summary_adapter.h>
#include <vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h>
#include <vespa/searchlib/index/dummyfileheadercontext.h>
#include <vespa/searchlib/transactionlog/nosyncproxy.h>
#include <vespa/vespalib/io/fileutil.h>
-#include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h>
using namespace config;
using namespace document;
diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
index 483e725927d..b9a04acb8da 100644
--- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp
@@ -3,7 +3,6 @@
#include <tests/proton/common/dummydbowner.h>
#include <vespa/searchcore/proton/attribute/flushableattribute.h>
#include <vespa/searchcore/proton/common/feedtoken.h>
-#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/common/statusreport.h>
#include <vespa/searchcore/proton/docsummary/summaryflushtarget.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
@@ -12,11 +11,9 @@
#include <vespa/searchcore/proton/matching/querylimiter.h>
#include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h>
#include <vespa/searchcore/proton/metrics/metricswireservice.h>
-#include <vespa/searchcore/proton/reference/document_db_reference_registry.h>
#include <vespa/searchcore/proton/reference/i_document_db_reference.h>
#include <vespa/searchcore/proton/server/bootstrapconfig.h>
#include <vespa/searchcore/proton/server/document_db_explorer.h>
-#include <vespa/searchcore/proton/server/documentdb.h>
#include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
#include <vespa/searchcore/proton/server/memoryconfigstore.h>
#include <vespa/searchcorespi/index/indexflushtarget.h>
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
index 7ebc2c36985..92a9dc06fff 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
+++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h
@@ -6,9 +6,7 @@
#include <vector>
#include <memory>
-namespace search {
-
-namespace grouping {
+namespace search::grouping {
/**
* A Grouping Context contains all grouping expressions that should be evaluated
@@ -115,6 +113,4 @@ public:
bool needRanking() const;
};
-} // namespace search::grouping
-} // namespace search
-
+}
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
index 7913043265a..48baba329ca 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
+++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
@@ -1,18 +1,18 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "groupingmanager.h"
+#include "groupingsession.h"
+#include "groupingcontext.h"
#include <vespa/searchlib/aggregation/fs4hit.h>
#include <vespa/searchlib/expression/attributenode.h>
-#include <vespa/searchcore/grouping/groupingsession.h>
#include <vespa/log/log.h>
LOG_SETUP(".groupingmanager");
-namespace search {
-namespace grouping {
+namespace search::grouping {
-using search::aggregation::Grouping;
-using search::attribute::IAttributeContext;
+using aggregation::Grouping;
+using attribute::IAttributeContext;
//-----------------------------------------------------------------------------
@@ -31,6 +31,10 @@ using search::expression::ConfigureStaticParams;
using search::aggregation::Grouping;
using search::aggregation::GroupingLevel;
+bool GroupingManager::empty() const {
+ return _groupingContext.getGroupingList().empty();
+}
+
void
GroupingManager::init(const IAttributeContext &attrCtx)
{
@@ -126,5 +130,4 @@ GroupingManager::convertToGlobalId(const search::IDocumentMetaStore &metaStore)
}
}
-} // namespace search::grouping
-} // namespace search
+}
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h
index 1a440bf9247..5793c8576e5 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.h
+++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.h
@@ -2,12 +2,16 @@
#pragma once
#include <vespa/searchlib/common/idocumentmetastore.h>
-#include <vespa/searchlib/aggregation/grouping.h>
-#include <vespa/searchcore/grouping/groupingcontext.h>
+#include <vespa/searchcommon/attribute/iattributecontext.h>
namespace search {
+ class RankedHit;
+ class BitVector;
+}
-namespace grouping {
+namespace search::grouping {
+
+class GroupingContext;
/**
* Wrapper class used to handle actual grouping. All input data is
@@ -37,14 +41,14 @@ public:
/**
* @return true if this manager is holding an empty grouping request.
**/
- bool empty() const { return _groupingContext.getGroupingList().empty(); }
+ bool empty() const;
/**
* Initialize underlying context with attribute bindings.
*
* @param attrCtx attribute context
**/
- void init(const search::attribute::IAttributeContext &attrCtx);
+ void init(const attribute::IAttributeContext &attrCtx);
/**
* Perform actual grouping on the given results.
@@ -65,7 +69,7 @@ public:
* @param binSize size of search result array
* @param overflow The unranked hits.
**/
- void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow);
+ void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const BitVector * overflow);
/**
* Merge another grouping context into the underlying context of
@@ -89,9 +93,7 @@ public:
*
* @param metaStore the attribute used to map from lid to gid.
**/
- void convertToGlobalId(const search::IDocumentMetaStore &metaStore);
+ void convertToGlobalId(const IDocumentMetaStore &metaStore);
};
-} // namespace search::grouping
-} // namespace search
-
+}
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp
index 6407a29175e..110d032205e 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp
+++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.cpp
@@ -1,11 +1,13 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "groupingsession.h"
+#include "groupingmanager.h"
+#include "groupingcontext.h"
+
#include <vespa/log/log.h>
LOG_SETUP(".groupingsession");
-namespace search {
-namespace grouping {
+namespace search::grouping {
using search::aggregation::Group;
using search::aggregation::Grouping;
@@ -16,8 +18,8 @@ GroupingSession::GroupingSession(const SessionId &sessionId,
GroupingContext & groupingContext,
const IAttributeContext &attrCtx)
: _sessionId(sessionId),
- _mgrContext(groupingContext),
- _groupingManager(_mgrContext),
+ _mgrContext(std::make_unique<GroupingContext>(groupingContext)),
+ _groupingManager(std::make_unique<GroupingManager>(*_mgrContext)),
_timeOfDoom(groupingContext.getTimeOfDoom())
{
init(groupingContext, attrCtx);
@@ -46,31 +48,30 @@ GroupingSession::init(GroupingContext & groupingContext, const IAttributeContext
_groupingMap[gp->getId()] = gp;
g = gp;
}
- _mgrContext.addGrouping(g);
+ _mgrContext->addGrouping(g);
}
- _groupingManager.init(attrCtx);
+ _groupingManager->init(attrCtx);
}
void
GroupingSession::prepareThreadContextCreation(size_t num_threads)
{
if (num_threads > 1) {
- _mgrContext.serialize(); // need copy of internal modified request
+ _mgrContext->serialize(); // need copy of internal modified request
}
}
GroupingContext::UP
GroupingSession::createThreadContext(size_t thread_id, const IAttributeContext &attrCtx)
{
- GroupingContext::UP ctx(new GroupingContext(_mgrContext));
+ GroupingContext::UP ctx(new GroupingContext(*_mgrContext));
if (thread_id == 0) {
- GroupingContext::GroupingList &groupingList = _mgrContext.getGroupingList();
+ GroupingContext::GroupingList &groupingList = _mgrContext->getGroupingList();
for (size_t i = 0; i < groupingList.size(); ++i) {
ctx->addGrouping(groupingList[i]);
}
} else {
- ctx->deserialize(_mgrContext.getResult().peek(),
- _mgrContext.getResult().size());
+ ctx->deserialize(_mgrContext->getResult().peek(), _mgrContext->getResult().size());
GroupingManager man(*ctx);
man.init(attrCtx);
}
@@ -97,5 +98,4 @@ GroupingSession::continueExecution(GroupingContext & groupingContext)
groupingContext.serialize();
}
-} // namespace search::grouping
-} // namespace search
+}
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingsession.h b/searchcore/src/vespa/searchcore/grouping/groupingsession.h
index 62867289be7..95a5332b417 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingsession.h
+++ b/searchcore/src/vespa/searchcore/grouping/groupingsession.h
@@ -1,16 +1,17 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include "groupingmanager.h"
-#include "groupingcontext.h"
#include "sessionid.h"
-#include <vespa/searchlib/aggregation/grouping.h>
+#include <vespa/searchlib/attribute/iattributemanager.h>
+#include <vespa/fastos/timestamp.h>
+#include <vector>
#include <map>
+namespace search::aggregation { class Grouping; }
+namespace search::grouping {
-namespace search {
-
-namespace grouping {
+class GroupingContext;
+class GroupingManager;
/**
* A grouping session represents the execution of a grouping expression with one
@@ -21,15 +22,15 @@ namespace grouping {
class GroupingSession
{
private:
- typedef std::shared_ptr<search::aggregation::Grouping> GroupingPtr;
- typedef std::map<uint32_t, GroupingPtr> GroupingMap;
- typedef std::vector<GroupingPtr> GroupingList;
+ using GroupingPtr = std::shared_ptr<aggregation::Grouping>;
+ using GroupingMap = std::map<uint32_t, GroupingPtr>;
+ using GroupingList = std::vector<GroupingPtr>;
- SessionId _sessionId;
- GroupingContext _mgrContext;
- GroupingManager _groupingManager;
- GroupingMap _groupingMap;
- fastos::TimeStamp _timeOfDoom;
+ SessionId _sessionId;
+ std::unique_ptr<GroupingContext> _mgrContext;
+ std::unique_ptr<GroupingManager> _groupingManager;
+ GroupingMap _groupingMap;
+ fastos::TimeStamp _timeOfDoom;
public:
typedef std::unique_ptr<GroupingSession> UP;
@@ -43,7 +44,7 @@ public:
**/
GroupingSession(const SessionId & sessionId,
GroupingContext & groupingContext,
- const search::attribute::IAttributeContext &attrCtx);
+ const attribute::IAttributeContext &attrCtx);
GroupingSession(const GroupingSession &) = delete;
GroupingSession &operator=(const GroupingSession &) = delete;
@@ -62,8 +63,7 @@ public:
* @param groupingContext The current grouping context.
* @param attrCtx attribute context.
**/
- void init(GroupingContext & groupingContext,
- const search::attribute::IAttributeContext &attrCtx);
+ void init(GroupingContext & groupingContext, const attribute::IAttributeContext &attrCtx);
/**
* This function is called to prepare for creation of individual
@@ -85,13 +85,12 @@ public:
* @param thread_id thread id
* @param attrCtx attribute context.
**/
- GroupingContext::UP createThreadContext(size_t thread_id,
- const search::attribute::IAttributeContext &attrCtx);
+ std::unique_ptr<GroupingContext> createThreadContext(size_t thread_id, const attribute::IAttributeContext &attrCtx);
/**
* Return the GroupingManager to use when performing grouping.
**/
- GroupingManager & getGroupingManager() { return _groupingManager; }
+ GroupingManager & getGroupingManager() { return *_groupingManager; }
/**
* Continue excuting a query given a context.
@@ -112,6 +111,4 @@ public:
fastos::TimeStamp getTimeOfDoom() const { return _timeOfDoom; }
};
-} // namespace search::grouping
-} // namespace search
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
index 4ece0c65593..1d5abcc2929 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -9,13 +9,13 @@
#include <vespa/vespalib/util/closure.h>
#include <vespa/vespalib/util/thread_bundle.h>
#include <vespa/searchcore/grouping/groupingmanager.h>
+#include <vespa/searchcore/grouping/groupingcontext.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/log/log.h>
LOG_SETUP(".proton.matching.match_thread");
-namespace proton {
-namespace matching {
+namespace proton::matching {
using search::queryeval::OptimizedAndNotForBlackListing;
using search::queryeval::SearchIterator;
@@ -402,5 +402,4 @@ MatchThread::run()
mergeDirector.dualMerge(thread_id, *resultContext->result, resultContext->groupingSource);
}
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
index 9287089c34e..cd01c330931 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
@@ -15,8 +15,7 @@
#include <vespa/searchlib/common/sortresults.h>
#include <vespa/searchlib/queryeval/hitcollector.h>
-namespace proton {
-namespace matching {
+namespace proton::matching {
/**
* Runs a single match thread and keeps track of local state.
@@ -111,5 +110,4 @@ public:
PartialResult::UP extract_result() { return std::move(resultContext->result); }
};
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
index 852176e4918..32775d7619a 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -7,6 +7,7 @@
#include "match_params.h"
#include "matcher.h"
#include "sessionmanager.h"
+#include <vespa/searchcore/grouping/groupingcontext.h>
#include <vespa/searchlib/engine/errorcodes.h>
#include <vespa/searchlib/engine/docsumrequest.h>
#include <vespa/searchlib/engine/searchrequest.h>
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
index 1e7acead748..e2c6affebe1 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
@@ -3,6 +3,8 @@
#include "result_processor.h"
#include "partial_result.h"
#include "sessionmanager.h"
+#include <vespa/searchcore/grouping/groupingmanager.h>
+#include <vespa/searchcore/grouping/groupingcontext.h>
#include <vespa/searchlib/common/docstamp.h>
#include <vespa/searchlib/uca/ucaconverter.h>
#include <vespa/searchlib/engine/searchreply.h>
@@ -15,8 +17,7 @@ using search::grouping::GroupingSession;
using search::grouping::GroupingContext;
using search::grouping::SessionId;
-namespace proton {
-namespace matching {
+namespace proton::matching {
ResultProcessor::Result::Result(std::unique_ptr<search::engine::SearchReply> reply, size_t numFs4Hits)
: _reply(std::move(reply)),
@@ -158,5 +159,4 @@ ResultProcessor::makeReply(PartialResultUP full_result)
return Result::UP(new Result(std::move(reply), numFs4Hits));
}
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
index 0a9b88c066a..a181c1660b7 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h
@@ -16,8 +16,7 @@ namespace search {
class IDocumentMetaStore;
}
-namespace proton {
-namespace matching {
+namespace proton::matching {
class SessionManager;
class PartialResult;
@@ -107,6 +106,4 @@ public:
std::unique_ptr<Result> makeReply(PartialResultUP full_result);
};
-} // namespace proton::matching
-} // namespace proton
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp
index f6fd7f7cc9d..e456ea5b5a2 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.cpp
@@ -1,14 +1,14 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "session_manager_explorer.h"
+#include "sessionmanager.h"
#include <vespa/vespalib/data/slime/slime.h>
using vespalib::slime::Inserter;
using vespalib::slime::Cursor;
using vespalib::StateExplorer;
-namespace proton {
-namespace matching {
+namespace proton::matching {
namespace {
@@ -59,5 +59,4 @@ SessionManagerExplorer::get_child(vespalib::stringref name) const
return std::unique_ptr<StateExplorer>(nullptr);
}
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h
index 31dabc5a887..b8acfba0342 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h
@@ -2,11 +2,11 @@
#pragma once
-#include "sessionmanager.h"
#include <vespa/vespalib/net/state_explorer.h>
-namespace proton {
-namespace matching {
+namespace proton::matching {
+
+class SessionManager;
/**
* Class used to explore the state of a session manager
@@ -20,8 +20,7 @@ public:
SessionManagerExplorer(const SessionManager &manager) : _manager(manager) {}
virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
virtual std::vector<vespalib::string> get_children_names() const override;
- virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override;
+ virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override;
};
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 2156d853ac4..6773798d2d7 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -21,7 +21,6 @@
#include <vespa/searchcore/proton/common/doctypename.h>
#include <vespa/searchcore/proton/common/monitored_refcount.h>
-#include <vespa/searchcore/proton/matching/sessionmanager.h>
#include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h>
#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h>
#include <vespa/searchcore/proton/persistenceengine/bucket_guard.h>
@@ -44,6 +43,8 @@ class IDocumentDBOwner;
class MetricsWireService;
class StatusReport;
+namespace matching { class SessionManager; }
+
/**
* The document database contains all the necessary structures required per
* document type. It has an internal single-threaded Executor to process input
@@ -110,7 +111,7 @@ private:
ProtonConfig::Summary _protonSummaryCfg;
ProtonConfig::Index _protonIndexCfg;
ConfigStore::UP _config_store;
- matching::SessionManager::SP _sessionManager; // TODO: This should not have to be a shared pointer.
+ std::shared_ptr<matching::SessionManager> _sessionManager; // TODO: This should not have to be a shared pointer.
MetricsWireService &_metricsWireService;
MetricsUpdateHook _metricsHook;
vespalib::VarHolder<IFeedView::SP> _feedView;
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
index c4ff03de6a1..c89a63b95c0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
@@ -60,8 +60,8 @@ public:
};
private:
- typedef vespa::config::search::AttributesConfig AttributesConfig;
- typedef FastAccessDocSubDBConfigurer Configurer;
+ using AttributesConfig = vespa::config::search::AttributesConfig;
+ using Configurer = FastAccessDocSubDBConfigurer;
const bool _hasAttributes;
const bool _fastAccessAttributesOnly;
@@ -81,7 +81,8 @@ private:
void initFeedView(const IAttributeWriter::SP &writer, const DocumentDBConfig &configSnapshot);
protected:
- typedef StoreOnlyDocSubDB Parent;
+ using Parent = StoreOnlyDocSubDB;
+ using SessionManagerSP = std::shared_ptr<matching::SessionManager>;
const bool _addMetrics;
MetricsWireService &_metricsWireService;
@@ -110,7 +111,7 @@ public:
void setup(const DocumentSubDbInitializerResult &initResult) override;
void initViews(const DocumentDBConfig &configSnapshot,
- const matching::SessionManager::SP &sessionManager) override;
+ const SessionManagerSP &sessionManager) override;
IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot,
const DocumentDBConfig &oldConfigSnapshot,
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
index 0f38e48a8e2..3162f9a1c45 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -33,7 +33,7 @@ using matching::SessionManager;
MatchView::MatchView(const Matchers::SP &matchers,
const IndexSearchable::SP &indexSearchable,
const IAttributeManager::SP &attrMgr,
- const SessionManager::SP &sessionMgr,
+ const SessionManagerSP &sessionMgr,
const IDocumentMetaStoreContext::SP &metaStore,
DocIdLimit &docIdLimit)
: _matchers(matchers),
@@ -44,6 +44,7 @@ MatchView::MatchView(const Matchers::SP &matchers,
_docIdLimit(docIdLimit)
{ }
+MatchView::~MatchView() { }
Matcher::SP
MatchView::getMatcher(const vespalib::string & rankProfile) const
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h
index 5207bce9288..511048f536f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h
@@ -8,17 +8,20 @@
#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
#include <vespa/searchcore/proton/matching/match_context.h>
#include <vespa/searchcore/proton/matching/matcher.h>
-#include <vespa/searchcore/proton/matching/sessionmanager.h>
#include <vespa/searchcorespi/index/indexsearchable.h>
#include <vespa/searchlib/attribute/attributevector.h>
namespace proton {
+namespace matching {
+ class SessionManager;
+}
class MatchView {
+ using SessionManagerSP = std::shared_ptr<matching::SessionManager>;
Matchers::SP _matchers;
searchcorespi::IndexSearchable::SP _indexSearchable;
IAttributeManager::SP _attrMgr;
- matching::SessionManager::SP _sessionMgr;
+ SessionManagerSP _sessionMgr;
IDocumentMetaStoreContext::SP _metaStore;
DocIdLimit &_docIdLimit;
@@ -34,14 +37,15 @@ public:
MatchView(const Matchers::SP &matchers,
const searchcorespi::IndexSearchable::SP &indexSearchable,
const IAttributeManager::SP &attrMgr,
- const matching::SessionManager::SP &sessionMgr,
+ const SessionManagerSP &sessionMgr,
const IDocumentMetaStoreContext::SP &metaStore,
DocIdLimit &docIdLimit);
+ ~MatchView();
const Matchers::SP & getMatchers() const { return _matchers; }
const searchcorespi::IndexSearchable::SP & getIndexSearchable() const { return _indexSearchable; }
const IAttributeManager::SP & getAttributeManager() const { return _attrMgr; }
- const matching::SessionManager::SP & getSessionManager() const { return _sessionMgr; }
+ const SessionManagerSP & getSessionManager() const { return _sessionMgr; }
const IDocumentMetaStoreContext::SP & getDocumentMetaStore() const { return _metaStore; }
DocIdLimit & getDocIdLimit() const { return _docIdLimit; }
@@ -62,4 +66,3 @@ public:
};
} // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index 7bd6c1ef100..fa3ded56c97 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -21,8 +21,6 @@
#include <vespa/searchcorespi/index/iindexmanager.h>
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
#include <vespa/vespalib/util/varholder.h>
-#include <memory>
-#include <vector>
namespace proton {
@@ -126,7 +124,7 @@ public:
void
initViews(const DocumentDBConfig &configSnapshot,
- const matching::SessionManager::SP &sessionManager) override;
+ const SessionManagerSP &sessionManager) override;
IReprocessingTask::List
applyConfig(const DocumentDBConfig &newConfigSnapshot,
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.h b/searchcore/src/vespa/searchcore/proton/server/searchview.h
index cb1cd7670bc..186d6154706 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.h
@@ -11,6 +11,7 @@ namespace proton {
class SearchView : public ISearchHandler
{
public:
+ using SessionManagerSP = std::shared_ptr<matching::SessionManager>;
using IndexSearchable = searchcorespi::IndexSearchable;
using InternalDocsumReply = std::pair<std::unique_ptr<DocsumReply>, bool>;
typedef std::shared_ptr<SearchView> SP;
@@ -25,7 +26,7 @@ public:
const Matchers::SP & getMatchers() const { return _matchView->getMatchers(); }
const IndexSearchable::SP & getIndexSearchable() const { return _matchView->getIndexSearchable(); }
const IAttributeManager::SP & getAttributeManager() const { return _matchView->getAttributeManager(); }
- const matching::SessionManager::SP & getSessionManager() const { return _matchView->getSessionManager(); }
+ const SessionManagerSP & getSessionManager() const { return _matchView->getSessionManager(); }
const IDocumentMetaStoreContext::SP & getDocumentMetaStore() const { return _matchView->getDocumentMetaStore(); }
DocIdLimit &getDocIdLimit() const { return _matchView->getDocIdLimit(); }
matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const { return _matchView->getMatcherStats(rankProfile); }
diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
index f855f51af42..6406ce6d4a1 100644
--- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
+++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
@@ -304,7 +304,7 @@ template <typename FixtureType>
void verify_get_string_from_enum_is_mapped(FixtureType& f) {
EnumHandle handle{};
ASSERT_TRUE(f.target_attr->findEnum("foo", handle));
- const char* from_enum = f.imported_attr->getStringFromEnum(handle);
+ const char* from_enum = f.get_imported_attr()->getStringFromEnum(handle);
ASSERT_TRUE(from_enum != nullptr);
EXPECT_EQUAL(vespalib::string("foo"), vespalib::string(from_enum));
}
@@ -438,7 +438,7 @@ struct MockAttributeVector : NotImplementedAttribute {
long _return_value{1234};
MockAttributeVector()
- : NotImplementedAttribute("mock", Config(BasicType::INT32)) {
+ : NotImplementedAttribute("mock", Config(BasicType::STRING)) {
}
void set_received_args(DocId doc_id, void* ser_to,
@@ -475,44 +475,65 @@ struct MockBlobConverter : common::BlobConverter {
}
};
-struct SerializeFixture : Fixture {
+template <typename BaseFixture>
+struct SerializeFixture : BaseFixture {
std::shared_ptr<MockAttributeVector> mock_target;
MockBlobConverter mock_converter;
- SerializeFixture()
- : Fixture(),
- mock_target(std::make_shared<MockAttributeVector>())
- {
- reset_with_new_target_attr(mock_target);
+ SerializeFixture() : mock_target(std::make_shared<MockAttributeVector>()) {
+ this->reset_with_new_target_attr(mock_target);
+ mock_target->setCommittedDocIdLimit(8); // Target LID of 7 is highest used by ref attribute. Limit is +1.
}
+ ~SerializeFixture() override;
};
-TEST_F("onSerializeForAscendingSort() is forwarded to target vector", SerializeFixture) {
+template <typename BaseFixture>
+SerializeFixture<BaseFixture>::~SerializeFixture() {}
+
+template <typename FixtureT>
+void check_onSerializeForAscendingSort_is_forwarded_with_remapped_lid() {
+ FixtureT f;
int dummy_tag;
void* ser_to = &dummy_tag;
EXPECT_EQUAL(f.mock_target->_return_value,
- f.imported_attr->serializeForAscendingSort(
- DocId(10), ser_to, 777, &f.mock_converter));
+ f.get_imported_attr()->serializeForAscendingSort(
+ DocId(4), ser_to, 777, &f.mock_converter)); // child lid 4 -> parent lid 7
EXPECT_TRUE(f.mock_target->_ascending_called);
- EXPECT_EQUAL(DocId(10), f.mock_target->_doc_id);
+ EXPECT_EQUAL(DocId(7), f.mock_target->_doc_id);
EXPECT_EQUAL(ser_to, f.mock_target->_ser_to);
EXPECT_EQUAL(777, f.mock_target->_available);
EXPECT_EQUAL(&f.mock_converter, f.mock_target->_bc);
}
-TEST_F("onSerializeForDescendingSort() is forwarded to target vector", SerializeFixture) {
+TEST("onSerializeForAscendingSort() is forwarded with remapped LID to target vector") {
+ TEST_DO(check_onSerializeForAscendingSort_is_forwarded_with_remapped_lid<
+ SerializeFixture<SingleStringAttrFixture>>());
+ TEST_DO(check_onSerializeForAscendingSort_is_forwarded_with_remapped_lid<
+ SerializeFixture<ReadGuardSingleStringAttrFixture>>());
+}
+
+template <typename FixtureT>
+void check_onSerializeForDescendingSort_is_forwarded_with_remapped_lid() {
+ FixtureT f;
int dummy_tag;
void* ser_to = &dummy_tag;
EXPECT_EQUAL(f.mock_target->_return_value,
- f.imported_attr->serializeForDescendingSort(
- DocId(20), ser_to, 555, &f.mock_converter));
+ f.get_imported_attr()->serializeForDescendingSort(
+ DocId(2), ser_to, 555, &f.mock_converter)); // child lid 2 -> parent lid 3
EXPECT_TRUE(f.mock_target->_descending_called);
- EXPECT_EQUAL(DocId(20), f.mock_target->_doc_id);
+ EXPECT_EQUAL(DocId(3), f.mock_target->_doc_id);
EXPECT_EQUAL(ser_to, f.mock_target->_ser_to);
EXPECT_EQUAL(555, f.mock_target->_available);
EXPECT_EQUAL(&f.mock_converter, f.mock_target->_bc);
}
+TEST("onSerializeForDescendingSort() is forwarded with remapped LID to target vector") {
+ TEST_DO(check_onSerializeForDescendingSort_is_forwarded_with_remapped_lid<
+ SerializeFixture<SingleStringAttrFixture>>());
+ TEST_DO(check_onSerializeForDescendingSort_is_forwarded_with_remapped_lid<
+ SerializeFixture<ReadGuardSingleStringAttrFixture>>());
+}
+
} // attribute
} // search
diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
index 5811ca9cc4d..c9e0757ee16 100644
--- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
+++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
@@ -197,6 +197,7 @@ TEST_F("Non-strict iterator unpacks target match data for weighted set hit", Wse
TEST_F("Strict iterator is marked as strict", Fixture) {
auto ctx = f.create_context(word_term("5678"));
+ ctx->fetchPostings(true);
TermFieldMatchData match;
auto iter = f.create_strict_iterator(*ctx, match);
@@ -218,11 +219,12 @@ struct SingleValueFixture : Fixture {
TEST_F("Strict iterator seeks to first available hit LID", SingleValueFixture) {
auto ctx = f.create_context(word_term("5678"));
+ ctx->fetchPostings(true);
TermFieldMatchData match;
auto iter = f.create_strict_iterator(*ctx, match);
EXPECT_FALSE(iter->isAtEnd());
- EXPECT_EQUAL(iter->beginId(), iter->getDocId());
+ EXPECT_EQUAL(DocId(3), iter->getDocId());
EXPECT_FALSE(iter->seek(DocId(1)));
EXPECT_FALSE(iter->isAtEnd());
@@ -243,6 +245,7 @@ TEST_F("Strict iterator seeks to first available hit LID", SingleValueFixture) {
TEST_F("Strict iterator unpacks target match data for single value hit", SingleValueFixture) {
auto ctx = f.create_context(word_term("5678"));
+ ctx->fetchPostings(true);
TermFieldMatchData match;
auto iter = f.create_strict_iterator(*ctx, match);
@@ -254,6 +257,7 @@ TEST_F("Strict iterator unpacks target match data for single value hit", SingleV
TEST_F("Strict iterator unpacks target match data for array hit", ArrayValueFixture) {
auto ctx = f.create_context(word_term("1234"));
+ ctx->fetchPostings(true);
TermFieldMatchData match;
auto iter = f.create_strict_iterator(*ctx, match);
@@ -265,6 +269,7 @@ TEST_F("Strict iterator unpacks target match data for array hit", ArrayValueFixt
TEST_F("Strict iterator unpacks target match data for weighted set hit", WsetValueFixture) {
auto ctx = f.create_context(word_term("foo"));
+ ctx->fetchPostings(true);
TermFieldMatchData match;
auto iter = f.create_strict_iterator(*ctx, match);
@@ -275,6 +280,7 @@ TEST_F("Strict iterator unpacks target match data for weighted set hit", WsetVal
TEST_F("Strict iterator handles seek outside of LID space", ArrayValueFixture) {
auto ctx = f.create_context(word_term("1234"));
+ ctx->fetchPostings(true);
TermFieldMatchData match;
auto iter = f.create_strict_iterator(*ctx, match);
@@ -306,6 +312,7 @@ TEST_F("cmp(weight) performs GID mapping and forwards to target attribute", Wset
TEST_F("Multiple iterators can be created from the same context", SingleValueFixture) {
auto ctx = f.create_context(word_term("5678"));
+ ctx->fetchPostings(true);
TermFieldMatchData match1;
auto iter1 = f.create_strict_iterator(*ctx, match1);
diff --git a/searchlib/src/vespa/searchlib/aggregation/group.h b/searchlib/src/vespa/searchlib/aggregation/group.h
index b34bd0fc88a..c769b6c1d27 100644
--- a/searchlib/src/vespa/searchlib/aggregation/group.h
+++ b/searchlib/src/vespa/searchlib/aggregation/group.h
@@ -64,7 +64,7 @@ public:
size_t operator() (const ResultNode & arg) const { return arg.hash(); }
};
- typedef std::vector<GroupingLevel> GroupingLevelList;
+ using GroupingLevelList = std::vector<GroupingLevel>;
private:
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
index b6ff442ae94..270f4d51788 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp
@@ -135,14 +135,16 @@ long ImportedAttributeVector::onSerializeForAscendingSort(DocId doc,
void *serTo,
long available,
const common::BlobConverter *bc) const {
- return _target_attribute->serializeForAscendingSort(doc, serTo, available, bc);
+ return _target_attribute->serializeForAscendingSort(
+ _reference_attribute->getReferencedLid(doc), serTo, available, bc);
}
long ImportedAttributeVector::onSerializeForDescendingSort(DocId doc,
void *serTo,
long available,
const common::BlobConverter *bc) const {
- return _target_attribute->serializeForDescendingSort(doc, serTo, available, bc);
+ return _target_attribute->serializeForDescendingSort(
+ _reference_attribute->getReferencedLid(doc), serTo, available, bc);
}
namespace {
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
index 85f8e980026..563834e6cdb 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
@@ -80,5 +80,19 @@ uint32_t ImportedAttributeVectorReadGuard::get(DocId docId, WeightedEnum *buffer
return _target_attribute->get(getReferencedLid(docId), buffer, sz);
}
+long ImportedAttributeVectorReadGuard::onSerializeForAscendingSort(DocId doc,
+ void *serTo,
+ long available,
+ const common::BlobConverter *bc) const {
+ return _target_attribute->serializeForAscendingSort(getReferencedLid(doc), serTo, available, bc);
+}
+
+long ImportedAttributeVectorReadGuard::onSerializeForDescendingSort(DocId doc,
+ void *serTo,
+ long available,
+ const common::BlobConverter *bc) const {
+ return _target_attribute->serializeForDescendingSort(getReferencedLid(doc), serTo, available, bc);
+}
+
}
}
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
index 81a3d24b6cf..f4db2b538d5 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
@@ -48,6 +48,11 @@ public:
virtual uint32_t get(DocId docId, WeightedString *buffer, uint32_t sz) const override;
virtual uint32_t get(DocId docId, WeightedConstChar *buffer, uint32_t sz) const override;
virtual uint32_t get(DocId docId, WeightedEnum *buffer, uint32_t sz) const override;
+protected:
+ virtual long onSerializeForAscendingSort(DocId doc, void * serTo, long available,
+ const common::BlobConverter * bc) const override;
+ virtual long onSerializeForDescendingSort(DocId doc, void * serTo, long available,
+ const common::BlobConverter * bc) const override;
};
}
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
index c431b956c1d..b0ac12ce8f9 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
@@ -7,6 +7,19 @@
#include <vespa/searchcommon/attribute/search_context_params.h>
#include <vespa/searchlib/fef/fef.h>
#include <vespa/searchlib/query/queryterm.h>
+#include <vespa/searchlib/queryeval/emptysearch.h>
+#include "dociditerator.h"
+
+using search::datastore::EntryRef;
+using search::queryeval::EmptySearch;
+using search::queryeval::SearchIterator;
+using search::attribute::ReferenceAttribute;
+using search::AttributeVector;
+
+using ReverseMappingRefs = ReferenceAttribute::ReverseMappingRefs;
+using ReverseMapping = ReferenceAttribute::ReverseMapping;
+using SearchContext = AttributeVector::SearchContext;
+
namespace search {
namespace attribute {
@@ -19,7 +32,9 @@ ImportedSearchContext::ImportedSearchContext(
_reference_attribute(*_imported_attribute.getReferenceAttribute()),
_target_attribute(*_imported_attribute.getTargetAttribute()),
_target_search_context(_target_attribute.getSearch(std::move(term), params)),
- _referencedLids(_reference_attribute.getReferencedLids())
+ _referencedLids(_reference_attribute.getReferencedLids()),
+ _merger(_reference_attribute.getCommittedDocIdLimit()),
+ _fetchPostingsDone(false)
{
}
@@ -32,6 +47,18 @@ unsigned int ImportedSearchContext::approximateHits() const {
std::unique_ptr<queryeval::SearchIterator>
ImportedSearchContext::createIterator(fef::TermFieldMatchData* matchData, bool strict) {
+ if (_merger.hasArray()) {
+ if (_merger.emptyArray()) {
+ return SearchIterator::UP(new EmptySearch());
+ } else {
+ using Posting = btree::BTreeKeyData<uint32_t, int32_t>;
+ using DocIt = DocIdIterator<Posting>;
+ DocIt postings;
+ auto array = _merger.getArray();
+ postings.set(&array[0], &array[array.size()]);
+ return std::make_unique<AttributePostingListIteratorT<DocIt>>(true, matchData, postings);
+ }
+ }
if (!strict) {
return std::make_unique<AttributeIteratorT<ImportedSearchContext>>(*this, matchData);
} else {
@@ -39,9 +66,112 @@ ImportedSearchContext::createIterator(fef::TermFieldMatchData* matchData, bool s
}
}
+namespace {
+
+struct WeightedRef {
+ EntryRef revMapIdx;
+ int32_t weight;
+
+ WeightedRef(EntryRef revMapIdx_, int32_t weight_)
+ : revMapIdx(revMapIdx_),
+ weight(weight_)
+ {
+ }
+};
+
+struct TargetResult {
+ std::vector<WeightedRef> weightedRefs;
+ size_t sizeSum;
+
+ TargetResult()
+ : weightedRefs(),
+ sizeSum(0)
+ {
+ }
+};
+
+TargetResult
+getTargetResult(ReverseMappingRefs reverseMappingRefs,
+ const ReverseMapping &reverseMapping,
+ SearchContext &target_search_context,
+ uint32_t committedDocIdLimit)
+{
+ TargetResult targetResult;
+ fef::TermFieldMatchData matchData;
+ auto targetItr = target_search_context.createIterator(&matchData, true);
+ uint32_t docIdLimit = reverseMappingRefs.size();
+ if (docIdLimit > committedDocIdLimit) {
+ docIdLimit = committedDocIdLimit;
+ }
+ uint32_t lid = 1;
+ targetItr->initRange(1, docIdLimit);
+ while (lid < docIdLimit) {
+ if (targetItr->seek(lid)) {
+ EntryRef revMapIdx = reverseMappingRefs[lid];
+ if (revMapIdx.valid()) {
+ uint32_t size = reverseMapping.frozenSize(revMapIdx);
+ targetResult.sizeSum += size;
+ targetItr->unpack(lid);
+ int32_t weight = matchData.getWeight();
+ targetResult.weightedRefs.emplace_back(revMapIdx, weight);
+ }
+ ++lid;
+ } else {
+ ++lid;
+ uint32_t nextLid = targetItr->getDocId();
+ if (nextLid > lid) {
+ lid = nextLid;
+ }
+ }
+ }
+ return targetResult;
+}
+
+class ReverseMappingPostingList
+{
+ const ReverseMapping &_reverseMapping;
+ EntryRef _revMapIdx;
+ int32_t _weight;
+public:
+ ReverseMappingPostingList(const ReverseMapping &reverseMapping, EntryRef revMapIdx, int32_t weight)
+ : _reverseMapping(reverseMapping),
+ _revMapIdx(revMapIdx),
+ _weight(weight)
+ {
+ }
+ ~ReverseMappingPostingList() { }
+ template <typename Func>
+ void foreach(Func func) const {
+ int32_t weight = _weight;
+ _reverseMapping.foreach_frozen_key(_revMapIdx, [func, weight](uint32_t lid) { func(lid, weight); });
+ }
+};
+
+}
+
+void ImportedSearchContext::makeMergedPostings()
+{
+ uint32_t committedTargetDocIdLimit = _target_attribute.getCommittedDocIdLimit();
+ std::atomic_thread_fence(std::memory_order_acquire);
+ TargetResult targetResult(getTargetResult(_reference_attribute.getReverseMappingRefs(),
+ _reference_attribute.getReverseMapping(),
+ *_target_search_context,
+ committedTargetDocIdLimit));
+ _merger.reserveArray(targetResult.weightedRefs.size(), targetResult.sizeSum);
+ const auto &reverseMapping = _reference_attribute.getReverseMapping();
+ for (const auto &weightedRef : targetResult.weightedRefs) {
+ _merger.addToArray(ReverseMappingPostingList(reverseMapping, weightedRef.revMapIdx, weightedRef.weight));
+ }
+ _merger.merge();
+}
+
void ImportedSearchContext::fetchPostings(bool strict) {
- (void)strict;
- // Imported attributes do not have posting lists (at least not currently), so this is a no-op.
+ assert(!_fetchPostingsDone);
+ _fetchPostingsDone = true;
+ _target_search_context->fetchPostings(strict);
+ if (strict) {
+ makeMergedPostings();
+ }
}
bool ImportedSearchContext::valid() const {
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
index ce6642cd93f..9be4578fac0 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
@@ -5,6 +5,7 @@
#include "attributevector.h"
#include <vespa/searchcommon/attribute/i_search_context.h>
#include <vespa/vespalib/util/arrayref.h>
+#include <vespa/searchlib/attribute/posting_list_merger.h>
#include <memory>
namespace search {
@@ -34,6 +35,10 @@ class ImportedSearchContext : public ISearchContext {
const AttributeVector& _target_attribute;
std::unique_ptr<AttributeVector::SearchContext> _target_search_context;
ReferencedLids _referencedLids;
+ PostingListMerger<int32_t> _merger;
+ bool _fetchPostingsDone;
+
+ void makeMergedPostings();
public:
ImportedSearchContext(std::unique_ptr<QueryTermSimple> term,
const SearchContextParams& params,
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
index 53b70c1da1d..4ee277b8733 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h
@@ -36,6 +36,7 @@ public:
btree::BTreeDefaultTraits,
btree::NoAggrCalc>;
using ReferencedLids = ReferenceMappings::ReferencedLids;
+ using ReverseMappingRefs = ReferenceMappings::ReverseMappingRefs;
private:
ReferenceStore _store;
ReferenceStoreIndices _indices;
@@ -74,6 +75,9 @@ public:
std::shared_ptr<IGidToLidMapperFactory> getGidToLidMapperFactory() const { return _gidToLidMapperFactory; }
ReferencedLids getReferencedLids() const { return _referenceMappings.getReferencedLids(); }
DocId getReferencedLid(DocId doc) const { return _referenceMappings.getReferencedLid(doc); }
+ ReverseMappingRefs getReverseMappingRefs() const { return _referenceMappings.getReverseMappingRefs(); }
+ const ReverseMapping &getReverseMapping() const { return _referenceMappings.getReverseMapping(); }
+
void notifyGidToLidChange(const GlobalId &gid, DocId referencedLid);
void populateReferencedLids();
virtual void clearDocs(DocId lidLow, DocId lidLimit) override;
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp
index 03acf6f2167..4edd9d45e60 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp
@@ -9,6 +9,7 @@ namespace search::attribute {
ReferenceMappings::ReferenceMappings(GenerationHolder &genHolder)
: _reverseMappingIndices(genHolder),
+ _referencedLidLimit(0),
_reverseMapping(),
_referencedLids(genHolder)
{
@@ -38,7 +39,6 @@ ReferenceMappings::syncForwardMapping(const Reference &entry)
{ referencedLids[lid] = referencedLid; });
}
-
void
ReferenceMappings::syncReverseMappingIndices(const Reference &entry)
{
@@ -46,6 +46,10 @@ ReferenceMappings::syncReverseMappingIndices(const Reference &entry)
if (referencedLid != 0u) {
_reverseMappingIndices.ensure_size(referencedLid + 1);
_reverseMappingIndices[referencedLid] = entry.revMapIdx();
+ if (referencedLid >= _referencedLidLimit) {
+ std::atomic_thread_fence(std::memory_order_release);
+ _referencedLidLimit = referencedLid + 1;
+ }
}
}
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h
index 631a610f773..3190e1b5a83 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h
+++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h
@@ -4,6 +4,7 @@
#include <vespa/searchlib/btree/btreestore.h>
#include <vespa/searchlib/common/rcuvector.h>
+#include <atomic>
namespace search::attribute {
@@ -27,6 +28,8 @@ class ReferenceMappings
// Vector containing references to trees of lids referencing given
// referenced lid.
ReverseMappingIndices _reverseMappingIndices;
+ // limit for referenced lid when accessing _reverseMappingIndices
+ uint32_t _referencedLidLimit;
// Store of B-Trees, used to map from gid or referenced lid to
// referencing lids.
ReverseMapping _reverseMapping;
@@ -38,6 +41,7 @@ class ReferenceMappings
public:
using ReferencedLids = vespalib::ConstArrayRef<uint32_t>;
+ using ReverseMappingRefs = vespalib::ConstArrayRef<EntryRef>;
ReferenceMappings(GenerationHolder &genHolder);
@@ -76,6 +80,12 @@ public:
ReferencedLids getReferencedLids() const { return ReferencedLids(&_referencedLids[0], _referencedLids.size()); }
uint32_t getReferencedLid(uint32_t doc) const { return _referencedLids[doc]; }
+ ReverseMappingRefs getReverseMappingRefs() const {
+ uint32_t referencedLidLimit = _referencedLidLimit;
+ std::atomic_thread_fence(std::memory_order_acquire);
+ return ReverseMappingRefs(&_reverseMappingIndices[0], referencedLidLimit);
+ }
+ const ReverseMapping &getReverseMapping() const { return _reverseMapping; }
};
template <typename FunctionType>
diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp
index 0ff4de00177..bab047827ad 100644
--- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.cpp
@@ -15,6 +15,12 @@ EmptySearch::doUnpack(uint32_t)
{
}
+EmptySearch::Trinary
+EmptySearch::is_strict() const
+{
+ return Trinary::True;
+}
+
EmptySearch::EmptySearch()
: SearchIterator()
{
diff --git a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h
index c03c533deb1..12d7430922c 100644
--- a/searchlib/src/vespa/searchlib/queryeval/emptysearch.h
+++ b/searchlib/src/vespa/searchlib/queryeval/emptysearch.h
@@ -16,6 +16,7 @@ protected:
SearchIterator::initRange(begin, end);
setAtEnd();
}
+ virtual Trinary is_strict() const override;
public:
EmptySearch();
diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp
index a0553625c8c..3830d3b71cb 100644
--- a/storage/src/tests/storageserver/documentapiconvertertest.cpp
+++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp
@@ -13,7 +13,6 @@
#include <vespa/document/bucket/bucketidfactory.h>
#include <vespa/config/subscription/configuri.h>
#include <vespa/vespalib/testkit/test_kit.h>
-#include <climits>
using document::DataType;
using document::DocIdString;
@@ -78,15 +77,12 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DocumentApiConverterTest);
void DocumentApiConverterTest::testPut()
{
- Document::SP
- doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
+ Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
documentapi::PutDocumentMessage putmsg(doc);
putmsg.setTimestamp(1234);
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(putmsg, _repo);
-
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(putmsg, _repo);
api::PutCommand* pc = dynamic_cast<api::PutCommand*>(cmd.get());
CPPUNIT_ASSERT(pc);
@@ -100,26 +96,23 @@ void DocumentApiConverterTest::testPut()
api::PutReply* pr = dynamic_cast<api::PutReply*>(rep.get());
CPPUNIT_ASSERT(pr);
- std::unique_ptr<mbus::Message> mbusmsg =
- _converter->toDocumentAPI(*pc, _repo);
+ std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(*pc, _repo);
documentapi::PutDocumentMessage* mbusput = dynamic_cast<documentapi::PutDocumentMessage*>(mbusmsg.get());
CPPUNIT_ASSERT(mbusput);
- CPPUNIT_ASSERT(mbusput->getDocument().get() == doc.get());
+ CPPUNIT_ASSERT(mbusput->getDocumentSP().get() == doc.get());
CPPUNIT_ASSERT(mbusput->getTimestamp() == 1234);
};
void DocumentApiConverterTest::testForwardedPut()
{
- Document::SP
- doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
+ Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
documentapi::PutDocumentMessage* putmsg = new documentapi::PutDocumentMessage(doc);
std::unique_ptr<mbus::Reply> reply(((documentapi::DocumentMessage*)putmsg)->createReply());
reply->setMessage(std::unique_ptr<mbus::Message>(putmsg));
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(*putmsg, _repo);
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(*putmsg, _repo);
((storage::api::PutCommand*)cmd.get())->setTimestamp(1234);
std::unique_ptr<storage::api::StorageReply> rep = cmd->makeReply();
@@ -132,8 +125,7 @@ void DocumentApiConverterTest::testForwardedPut()
void DocumentApiConverterTest::testRemove()
{
documentapi::RemoveDocumentMessage removemsg(document::DocumentId(document::DocIdString("test", "test")));
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(removemsg, _repo);
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(removemsg, _repo);
api::RemoveCommand* rc = dynamic_cast<api::RemoveCommand*>(cmd.get());
@@ -148,8 +140,7 @@ void DocumentApiConverterTest::testRemove()
api::RemoveReply* pr = dynamic_cast<api::RemoveReply*>(rep.get());
CPPUNIT_ASSERT(pr);
- std::unique_ptr<mbus::Message> mbusmsg =
- _converter->toDocumentAPI(*rc, _repo);
+ std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(*rc, _repo);
documentapi::RemoveDocumentMessage* mbusremove = dynamic_cast<documentapi::RemoveDocumentMessage*>(mbusmsg.get());
CPPUNIT_ASSERT(mbusremove);
@@ -159,11 +150,9 @@ void DocumentApiConverterTest::testRemove()
void DocumentApiConverterTest::testGet()
{
documentapi::GetDocumentMessage getmsg(
- document::DocumentId(document::DocIdString("test", "test")),
- "foo bar");
+ document::DocumentId(document::DocIdString("test", "test")), "foo bar");
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(getmsg, _repo);
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(getmsg, _repo);
api::GetCommand* rc = dynamic_cast<api::GetCommand*>(cmd.get());
@@ -174,17 +163,10 @@ void DocumentApiConverterTest::testGet()
void DocumentApiConverterTest::testCreateVisitor()
{
- documentapi::CreateVisitorMessage cv(
- "mylib",
- "myinstance",
- "control-dest",
- "data-dest");
+ documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest");
cv.setTimeRemaining(123456);
-
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(cv, _repo);
-
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo);
api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get());
CPPUNIT_ASSERT(pc);
@@ -197,17 +179,9 @@ void DocumentApiConverterTest::testCreateVisitor()
void DocumentApiConverterTest::testCreateVisitorHighTimeout()
{
- documentapi::CreateVisitorMessage cv(
- "mylib",
- "myinstance",
- "control-dest",
- "data-dest");
-
+ documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest");
cv.setTimeRemaining((uint64_t)std::numeric_limits<uint32_t>::max() + 1); // Will be INT_MAX
-
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(cv, _repo);
-
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo);
api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get());
CPPUNIT_ASSERT(pc);
@@ -215,65 +189,40 @@ void DocumentApiConverterTest::testCreateVisitorHighTimeout()
CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId());
CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), pc->getControlDestination());
CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), pc->getDataDestination());
- CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(),
- pc->getTimeout());
+ CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), pc->getTimeout());
}
void DocumentApiConverterTest::testCreateVisitorReplyNotReady()
{
- documentapi::CreateVisitorMessage cv(
- "mylib",
- "myinstance",
- "control-dest",
- "data-dest");
-
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(cv, _repo);
+ documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest");
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo);
CPPUNIT_ASSERT(cmd.get());
api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd);
-
api::CreateVisitorReply cvr(cvc);
cvr.setResult(api::ReturnCode(api::ReturnCode::NOT_READY, "not ready"));
std::unique_ptr<documentapi::CreateVisitorReply> reply(
- dynamic_cast<documentapi::CreateVisitorReply*>(
- cv.createReply().release()));
+ dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release()));
CPPUNIT_ASSERT(reply.get());
-
_converter->transferReplyState(cvr, *reply);
-
CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::ERROR_NODE_NOT_READY, reply->getError(0).getCode());
-
- CPPUNIT_ASSERT_EQUAL(document::BucketId(INT_MAX), reply->getLastBucket());
+ CPPUNIT_ASSERT_EQUAL(document::BucketId(std::numeric_limits<int>::max()), reply->getLastBucket());
}
void DocumentApiConverterTest::testCreateVisitorReplyLastBucket()
{
- documentapi::CreateVisitorMessage cv(
- "mylib",
- "myinstance",
- "control-dest",
- "data-dest");
-
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(cv, _repo);
+ documentapi::CreateVisitorMessage cv("mylib", "myinstance", "control-dest", "data-dest");
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo);
CPPUNIT_ASSERT(cmd.get());
api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd);
-
-
api::CreateVisitorReply cvr(cvc);
cvr.setLastBucket(document::BucketId(123));
-
-
std::unique_ptr<documentapi::CreateVisitorReply> reply(
- dynamic_cast<documentapi::CreateVisitorReply*>(
- cv.createReply().release()));
+ dynamic_cast<documentapi::CreateVisitorReply*>(cv.createReply().release()));
CPPUNIT_ASSERT(reply.get());
-
_converter->transferReplyState(cvr, *reply);
-
CPPUNIT_ASSERT_EQUAL(document::BucketId(123), reply->getLastBucket());
}
@@ -282,8 +231,7 @@ void DocumentApiConverterTest::testDestroyVisitor()
{
documentapi::DestroyVisitorMessage cv("myinstance");
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(cv, _repo);
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(cv, _repo);
api::DestroyVisitorCommand* pc = dynamic_cast<api::DestroyVisitorCommand*>(cmd.get());
@@ -302,8 +250,7 @@ DocumentApiConverterTest::testVisitorInfo()
vicmd.setBucketsCompleted(bucketsCompleted);
- std::unique_ptr<mbus::Message> mbusmsg =
- _converter->toDocumentAPI(vicmd, _repo);
+ std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(vicmd, _repo);
documentapi::VisitorInfoMessage* mbusvi = dynamic_cast<documentapi::VisitorInfoMessage*>(mbusmsg.get());
CPPUNIT_ASSERT(mbusvi);
@@ -323,8 +270,7 @@ DocumentApiConverterTest::testVisitorInfo()
void
DocumentApiConverterTest::testDocBlock()
{
- Document::SP
- doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
+ Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
char buffer[10000];
vdslib::WritableDocumentList docBlock(_repo, buffer, sizeof(buffer));
@@ -335,11 +281,9 @@ DocumentApiConverterTest::testDocBlock()
bucketId.setUsedBits(32);
api::DocBlockCommand dbcmd(bucketId, docBlock, std::shared_ptr<void>());
-
dbcmd.setTimeout(123456);
- std::unique_ptr<mbus::Message> mbusmsg =
- _converter->toDocumentAPI(dbcmd, _repo);
+ std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(dbcmd, _repo);
documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get());
CPPUNIT_ASSERT(mbusdb);
@@ -370,12 +314,10 @@ DocumentApiConverterTest::testDocBlockWithKeepTimeStamps()
{
CPPUNIT_ASSERT_EQUAL(dbcmd.keepTimeStamps(), false);
- std::unique_ptr<mbus::Message> mbusmsg =
- _converter->toDocumentAPI(dbcmd, _repo);
+ std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(dbcmd, _repo);
documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get());
CPPUNIT_ASSERT(mbusdb);
-
CPPUNIT_ASSERT_EQUAL(mbusdb->keepTimeStamps(), false);
}
@@ -383,12 +325,10 @@ DocumentApiConverterTest::testDocBlockWithKeepTimeStamps()
dbcmd.keepTimeStamps(true);
CPPUNIT_ASSERT_EQUAL(dbcmd.keepTimeStamps(), true);
- std::unique_ptr<mbus::Message> mbusmsg =
- _converter->toDocumentAPI(dbcmd, _repo);
+ std::unique_ptr<mbus::Message> mbusmsg = _converter->toDocumentAPI(dbcmd, _repo);
documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get());
CPPUNIT_ASSERT(mbusdb);
-
CPPUNIT_ASSERT_EQUAL(mbusdb->keepTimeStamps(), true);
}
@@ -399,8 +339,7 @@ void
DocumentApiConverterTest::testMultiOperation()
{
//create a document
- Document::SP
- doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
+ Document::SP doc(new Document(_html_type, DocumentId(DocIdString("test", "test"))));
document::BucketIdFactory fac;
document::BucketId bucketId = fac.getBucketId(doc->getId());
@@ -409,17 +348,13 @@ DocumentApiConverterTest::testMultiOperation()
{
documentapi::MultiOperationMessage momsg(_repo, bucketId, 10000);
- vdslib::WritableDocumentList operations(_repo, &(momsg.getBuffer()[0]),
- momsg.getBuffer().size());
+ vdslib::WritableDocumentList operations(_repo, &(momsg.getBuffer()[0]), momsg.getBuffer().size());
operations.addPut(*doc, 100);
-
momsg.setOperations(operations);
-
CPPUNIT_ASSERT(momsg.getBuffer().size() > 0);
// Convert it to Storage API
- std::unique_ptr<api::StorageCommand> stcmd =
- _converter->toStorageAPI(momsg, _repo);
+ std::unique_ptr<api::StorageCommand> stcmd = _converter->toStorageAPI(momsg, _repo);
api::MultiOperationCommand* mocmd = dynamic_cast<api::MultiOperationCommand*>(stcmd.get());
CPPUNIT_ASSERT(mocmd);
@@ -443,8 +378,7 @@ DocumentApiConverterTest::testMultiOperation()
mocmd.getOperations().addPut(*doc, 100);
// Convert it to documentapi
- std::unique_ptr<mbus::Message> mbmsg =
- _converter->toDocumentAPI(mocmd, _repo);
+ std::unique_ptr<mbus::Message> mbmsg = _converter->toDocumentAPI(mocmd, _repo);
documentapi::MultiOperationMessage* momsg = dynamic_cast<documentapi::MultiOperationMessage*>(mbmsg.get());
CPPUNIT_ASSERT(momsg);
@@ -473,33 +407,28 @@ DocumentApiConverterTest::testBatchDocumentUpdate()
{
document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1"));
- document::DocumentUpdate::SP update(
- new document::DocumentUpdate(_html_type, docId));
+ document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId));
updates.push_back(update);
}
{
document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2"));
- document::DocumentUpdate::SP update(
- new document::DocumentUpdate(_html_type, docId));
+ document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId));
updates.push_back(update);
}
{
document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3"));
- document::DocumentUpdate::SP update(
- new document::DocumentUpdate(_html_type, docId));
+ document::DocumentUpdate::SP update(new document::DocumentUpdate(_html_type, docId));
updates.push_back(update);
}
- std::shared_ptr<documentapi::BatchDocumentUpdateMessage> msg(
- new documentapi::BatchDocumentUpdateMessage(1234));
+ auto msg = std::make_shared<documentapi::BatchDocumentUpdateMessage>(1234);
for (std::size_t i = 0; i < updates.size(); ++i) {
msg->addUpdate(updates[i]);
}
- std::unique_ptr<storage::api::StorageCommand> cmd =
- _converter->toStorageAPI(*msg, _repo);
+ std::unique_ptr<storage::api::StorageCommand> cmd = _converter->toStorageAPI(*msg, _repo);
api::BatchDocumentUpdateCommand* batchCmd = dynamic_cast<api::BatchDocumentUpdateCommand*>(cmd.get());
CPPUNIT_ASSERT(batchCmd);
CPPUNIT_ASSERT_EQUAL(updates.size(), batchCmd->getUpdates().size());
diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp
index 974c756359f..3f1e9b69963 100644
--- a/storage/src/tests/visiting/visitormanagertest.cpp
+++ b/storage/src/tests/visiting/visitormanagertest.cpp
@@ -298,7 +298,7 @@ VisitorManagerTest::getMessagesAndReply(
switch (session.sentMessages[i]->getType()) {
case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT:
docs.push_back(static_cast<documentapi::PutDocumentMessage&>(
- *session.sentMessages[i]).getDocument());
+ *session.sentMessages[i]).getDocumentSP());
break;
case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT:
docIds.push_back(static_cast<documentapi::RemoveDocumentMessage&>(
diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp
index 85fbb6207bb..8abe7a3857d 100644
--- a/storage/src/tests/visiting/visitortest.cpp
+++ b/storage/src/tests/visiting/visitortest.cpp
@@ -333,26 +333,22 @@ VisitorTest::getMessagesAndReply(
{
vespalib::MonitorGuard guard(session.getMonitor());
CPPUNIT_ASSERT(!session.sentMessages.empty());
- std::unique_ptr<documentapi::DocumentMessage> msg(
- std::move(session.sentMessages.front()));
+ std::unique_ptr<documentapi::DocumentMessage> msg(std::move(session.sentMessages.front()));
session.sentMessages.pop_front();
CPPUNIT_ASSERT(msg->getPriority() < 16);
switch (msg->getType()) {
case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT:
docs.push_back(
- static_cast<documentapi::PutDocumentMessage&>(*msg)
- .getDocument());
+ static_cast<documentapi::PutDocumentMessage&>(*msg).getDocumentSP());
break;
case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT:
docIds.push_back(
- static_cast<documentapi::RemoveDocumentMessage&>(*msg)
- .getDocumentId());
+ static_cast<documentapi::RemoveDocumentMessage&>(*msg).getDocumentId());
break;
case documentapi::DocumentProtocol::MESSAGE_VISITORINFO:
infoMessages.push_back(
- static_cast<documentapi::VisitorInfoMessage&>(*msg)
- .getErrorMessage());
+ static_cast<documentapi::VisitorInfoMessage&>(*msg).getErrorMessage());
break;
default:
break;
diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp
index 7dc5581d44e..9df177a32dd 100644
--- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp
+++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp
@@ -36,7 +36,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg,
case DocumentProtocol::MESSAGE_PUTDOCUMENT:
{
documentapi::PutDocumentMessage& from(static_cast<documentapi::PutDocumentMessage&>(fromMsg));
- api::PutCommand::UP to(new api::PutCommand(document::BucketId(0), from.getDocument(), from.getTimestamp()));
+ api::PutCommand::UP to(new api::PutCommand(document::BucketId(0), from.getDocumentSP(), from.getTimestamp()));
to->setCondition(from.getCondition());
toMsg = std::move(to);
break;
@@ -44,7 +44,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg,
case DocumentProtocol::MESSAGE_UPDATEDOCUMENT:
{
documentapi::UpdateDocumentMessage& from(static_cast<documentapi::UpdateDocumentMessage&>(fromMsg));
- api::UpdateCommand::UP to(new api::UpdateCommand(document::BucketId(0), from.getDocumentUpdate(),
+ api::UpdateCommand::UP to(new api::UpdateCommand(document::BucketId(0), from.getDocumentUpdateSP(),
from.getNewTimestamp()));
to->setOldTimestamp(from.getOldTimestamp());
to->setCondition(from.getCondition());
diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp
index 2564c500cbc..f3595afe1e7 100644
--- a/storageserver/src/tests/storageservertest.cpp
+++ b/storageserver/src/tests/storageservertest.cpp
@@ -6,7 +6,6 @@
#include <vespa/document/base/testdocman.h>
#include <vespa/documentapi/documentapi.h>
#include <vespa/messagebus/rpcmessagebus.h>
-#include <fstream>
#include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h>
#include <vespa/messagebus/staticthrottlepolicy.h>
#include <vespa/messagebus/testlib/slobrok.h>
@@ -14,17 +13,15 @@
#include <vespa/storageapi/mbusprot/storagereply.h>
#include <vespa/storageapi/message/bucketsplitting.h>
#include <vespa/storageapi/message/state.h>
-#include <vespa/storage/common/nodestateupdater.h>
#include <vespa/storage/common/statusmetricconsumer.h>
-#include <vespa/memfilepersistence/memfile/memfilecache.h>
#include <tests/testhelper.h>
-#include <vespa/vdstestlib/cppunit/macros.h>
#include <tests/dummystoragelink.h>
#include <vespa/slobrok/sbmirror.h>
#include <vespa/storageserver/app/distributorprocess.h>
#include <vespa/storageserver/app/memfileservicelayerprocess.h>
#include <vespa/vespalib/util/exceptions.h>
#include <sys/time.h>
+#include <fstream>
#include <vespa/log/log.h>
LOG_SETUP(".storageservertest");
@@ -388,9 +385,8 @@ namespace {
if (msg->getType() == DocumentProtocol::MESSAGE_PUTDOCUMENT)
{
documentapi::PutDocumentMessage& putMsg(
- static_cast<documentapi::PutDocumentMessage&>(
- *msg));
- std::cerr << " - " << putMsg.getDocument()->getId();
+ static_cast<documentapi::PutDocumentMessage&>(*msg));
+ std::cerr << " - " << putMsg.getDocument().getId();
}
std::cerr << "\n";
}
@@ -419,8 +415,7 @@ namespace {
}
FastOS_Thread::Sleep(1);
}
- LOG(info, "Currently, we have received %u ok replies and have %u "
- "pending ones.",
+ LOG(info, "Currently, we have received %u ok replies and have %u pending ones.",
_processedOk, _currentPending);
}
};
diff --git a/travis/travis-build-cpp.sh b/travis/travis-build-cpp.sh
index 6320cd505ca..42dbf0e6467 100755
--- a/travis/travis-build-cpp.sh
+++ b/travis/travis-build-cpp.sh
@@ -7,7 +7,7 @@ BUILD_DIR=~/build
mkdir "${BUILD_DIR}"
-export CCACHE_SIZE="4G"
+export CCACHE_SIZE="1G"
export CCACHE_COMPRESS=1
NUM_THREADS=4