summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Marius Venstad <venstad@gmail.com>2019-09-30 09:57:25 +0200
committerJon Marius Venstad <venstad@gmail.com>2019-09-30 09:57:25 +0200
commitb2e4292a6be3dd1f838d9dc528ec0ea1fb7c48f3 (patch)
treec04e0aecda27e269638282766bc878dfbd37f2f0
parente8a11563f9b44187e0c78efc66536b35285c2a5c (diff)
Stop writing instance data, and removed unused code
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java29
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java16
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java167
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java8
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java96
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java16
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializer.java574
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java117
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java10
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java24
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializerTest.java223
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldCuratorDb.java687
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldMockCuratorDb.java22
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-instance.json534
16 files changed, 69 insertions, 2462 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index 83fb71422cb..6b297e51134 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -58,7 +58,7 @@ public class Application {
public Application(TenantAndApplicationId id, Instant now) {
this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Change.empty(), Change.empty(),
Optional.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
- new ApplicationMetrics(0, 0), Optional.empty(), OptionalLong.empty(), true, List.of());
+ new ApplicationMetrics(0, 0), Optional.empty(), OptionalLong.empty(), false, List.of());
}
// DO NOT USE! For serialization purposes, only.
@@ -87,33 +87,6 @@ public class Application {
::iterator);
}
- /** Returns an aggregate application, from the given instances, if at least one. */
- public static Optional<Application> aggregate(List<Instance> instances) {
- if (instances.isEmpty())
- return Optional.empty();
-
- Instance base = instances.stream()
- .filter(instance -> instance.id().instance().isDefault())
- .findFirst()
- .orElse(instances.iterator().next());
-
- return Optional.of(new Application(TenantAndApplicationId.from(base.id()), base.createdAt(), base.deploymentSpec(),
- base.validationOverrides(), base.change(), base.outstandingChange(),
- base.deploymentJobs().issueId(), base.ownershipIssueId(), base.owner(),
- base.majorVersion(), base.metrics(), base.pemDeployKey(),
- base.deploymentJobs().projectId(), base.deploymentJobs().deployedInternally(), instances));
- }
-
- /** Returns an old Instance representation of this and the given instance, for serialisation. */
- public Instance legacy(InstanceName instance) {
- Instance base = require(instance);
-
- return new Instance(base.id(), createdAt, deploymentSpec, validationOverrides, base.deployments(),
- new DeploymentJobs(projectId, base.deploymentJobs().jobStatus().values(), deploymentIssueId, internal),
- change, outstandingChange, ownershipIssueId, owner,
- majorVersion, metrics, pemDeployKey, base.rotations(), base.rotationStatus());
- }
-
public TenantAndApplicationId id() { return id; }
public Instant createdAt() { return createdAt; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index e58aed115ec..c012abff670 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -286,14 +286,12 @@ public class ApplicationController {
if ( ! id.instance().isTester()) // Only store the application permits for non-user applications.
accessControl.createApplication(id, credentials.get());
}
- List<Instance> instances = getApplication(TenantAndApplicationId.from(id)).map(application -> application.instances().values())
- .map(ArrayList::new)
- .orElse(new ArrayList<>());
- instances.add(new Instance(id, clock.instant()));
- LockedApplication application = new LockedApplication(Application.aggregate(instances).get(), lock);
- store(application);
- log.info("Created " + application);
- return application.get();
+ Application application = getApplication(TenantAndApplicationId.from(id)).orElse(new Application(TenantAndApplicationId.from(id),
+ clock.instant()));
+ LockedApplication locked = new LockedApplication(application, lock).withNewInstance(id.instance());
+ store(locked);
+ log.info("Created " + locked);
+ return locked.get();
}
}
@@ -755,7 +753,7 @@ public class ApplicationController {
controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal);
});
});
- curator.storeWithoutInstance(application.without(applicationId.instance()).get(), applicationId);
+ curator.storeWithoutInstance(application.without(applicationId.instance()).get());
log.info("Deleted " + application);
});
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
index 3f1bebfed48..f885b7a146e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
@@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.controller;
import com.google.common.collect.ImmutableMap;
import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
@@ -13,11 +11,7 @@ import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.application.ApplicationActivity;
import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
-import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
import com.yahoo.vespa.hosted.controller.application.Deployment;
@@ -26,20 +20,17 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.EndpointList;
import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
-import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
import java.time.Instant;
-import java.util.Collections;
-import java.util.Comparator;
+import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.OptionalInt;
import java.util.OptionalLong;
+import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -53,95 +44,40 @@ import java.util.stream.Collectors;
public class Instance {
private final ApplicationId id;
- private final Instant createdAt;
- private final DeploymentSpec deploymentSpec;
- private final ValidationOverrides validationOverrides;
private final Map<ZoneId, Deployment> deployments;
private final DeploymentJobs deploymentJobs;
- private final Change change;
- private final Change outstandingChange;
- private final Optional<IssueId> ownershipIssueId;
- private final Optional<User> owner;
- private final OptionalInt majorVersion;
- private final ApplicationMetrics metrics;
- private final Optional<String> pemDeployKey;
private final List<AssignedRotation> rotations;
private final RotationStatus rotationStatus;
- /** Creates an empty instance*/
- public Instance(ApplicationId id, Instant now) {
- this(id, now, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(),
- new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false),
- Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
- new ApplicationMetrics(0, 0),
- Optional.empty(), Collections.emptyList(), RotationStatus.EMPTY);
+ /** Creates an empty instance */
+ public Instance(ApplicationId id) {
+ this(id, Set.of(), new DeploymentJobs(List.of()),
+ List.of(), RotationStatus.EMPTY);
}
/** Creates an empty instance*/
- public Instance(ApplicationId id, List<Deployment> deployments, DeploymentJobs deploymentJobs,
+ public Instance(ApplicationId id, Collection<Deployment> deployments, DeploymentJobs deploymentJobs,
List<AssignedRotation> rotations, RotationStatus rotationStatus) {
- this(id,
- Instant.EPOCH, DeploymentSpec.empty, ValidationOverrides.empty,
- deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())),
- deploymentJobs,
- Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
- new ApplicationMetrics(0, 0), Optional.empty(),
- rotations, rotationStatus);
- }
-
- /** Used from persistence layer: Do not use */
- public Instance(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
- List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
- Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
- OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
- List<AssignedRotation> rotations, RotationStatus rotationStatus) {
- this(id, createdAt, deploymentSpec, validationOverrides,
- deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())),
- deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
- metrics, pemDeployKey, rotations, rotationStatus);
- }
-
- Instance(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
- Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
- Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
- OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
- List<AssignedRotation> rotations, RotationStatus rotationStatus) {
this.id = Objects.requireNonNull(id, "id cannot be null");
- this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
- this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
- this.validationOverrides = Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null");
- this.deployments = ImmutableMap.copyOf(Objects.requireNonNull(deployments, "deployments cannot be null"));
+ this.deployments = ImmutableMap.copyOf(Objects.requireNonNull(deployments, "deployments cannot be null").stream()
+ .collect(Collectors.toMap(Deployment::zone, Function.identity())));
this.deploymentJobs = Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null");
- this.change = Objects.requireNonNull(change, "change cannot be null");
- this.outstandingChange = Objects.requireNonNull(outstandingChange, "outstandingChange cannot be null");
- this.ownershipIssueId = Objects.requireNonNull(ownershipIssueId, "ownershipIssueId cannot be null");
- this.owner = Objects.requireNonNull(owner, "owner cannot be null");
- this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null");
- this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
- this.pemDeployKey = pemDeployKey;
this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null"));
this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null");
}
public Instance withJobPause(JobType jobType, OptionalLong pausedUntil) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange,
- ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
+ return new Instance(id, deployments.values(), deploymentJobs.withPause(jobType, pausedUntil),
rotations, rotationStatus);
}
- public Instance withJobCompletion(JobType jobType, JobStatus.JobRun completion,
- Optional<DeploymentJobs.JobError> jobError) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs.withCompletion(jobType, completion, jobError),
- change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics,
- pemDeployKey, rotations, rotationStatus);
+ public Instance withJobCompletion(JobType jobType, JobStatus.JobRun completion, Optional<DeploymentJobs.JobError> jobError) {
+ return new Instance(id, deployments.values(), deploymentJobs.withCompletion(jobType, completion, jobError),
+ rotations, rotationStatus);
}
public Instance withJobTriggering(JobType jobType, JobStatus.JobRun job) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs.withTriggering(jobType, job), change, outstandingChange,
- ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
+ return new Instance(id, deployments.values(), deploymentJobs.withTriggering(jobType, job),
rotations, rotationStatus);
}
@@ -168,7 +104,6 @@ public class Instance {
Deployment deployment = deployments.get(zone);
if (deployment == null) return this; // No longer deployed in this zone.
return with(deployment.withClusterInfo(clusterInfo));
-
}
public Instance recordActivityAt(Instant instant, ZoneId zone) {
@@ -190,28 +125,18 @@ public class Instance {
}
public Instance withoutDeploymentJob(JobType jobType) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs.without(jobType), change, outstandingChange,
- ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
+ return new Instance(id, deployments.values(), deploymentJobs.without(jobType),
rotations, rotationStatus);
}
- public Instance with(ApplicationMetrics metrics) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
- metrics, pemDeployKey, rotations, rotationStatus);
- }
-
public Instance with(List<AssignedRotation> assignedRotations) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
- metrics, pemDeployKey, assignedRotations, rotationStatus);
+ return new Instance(id, deployments.values(), deploymentJobs,
+ assignedRotations, rotationStatus);
}
public Instance with(RotationStatus rotationStatus) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
- metrics, pemDeployKey, rotations, rotationStatus);
+ return new Instance(id, deployments.values(), deploymentJobs,
+ rotations, rotationStatus);
}
private Instance with(Deployment deployment) {
@@ -221,30 +146,14 @@ public class Instance {
}
private Instance with(Map<ZoneId, Deployment> deployments) {
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments,
- deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
- metrics, pemDeployKey, rotations, rotationStatus);
+ return new Instance(id, deployments.values(), deploymentJobs,
+ rotations, rotationStatus);
}
public ApplicationId id() { return id; }
public InstanceName name() { return id.instance(); }
- public Instant createdAt() { return createdAt; }
-
- /**
- * Returns the last deployed deployment spec of this application,
- * or the empty deployment spec if it has never been deployed
- */
- public DeploymentSpec deploymentSpec() { return deploymentSpec; }
-
- /**
- * Returns the last deployed validation overrides of this application,
- * or the empty validation overrides if it has never been deployed
- * (or was deployed with an empty/missing validation overrides)
- */
- public ValidationOverrides validationOverrides() { return validationOverrides; }
-
/** Returns an immutable map of the current deployments of this */
public Map<ZoneId, Deployment> deployments() { return deployments; }
@@ -260,38 +169,6 @@ public class Instance {
public DeploymentJobs deploymentJobs() { return deploymentJobs; }
- /**
- * Returns base change for this application, i.e., the change that is deployed outside block windows.
- * This is empty when no change is currently under deployment.
- */
- public Change change() { return change; }
-
- /**
- * Returns whether this has an outstanding change (in the source repository), which
- * has currently not started deploying (because a deployment is (or was) already in progress
- */
- public Change outstandingChange() { return outstandingChange; }
-
- /** Returns ID of the last ownership issue filed for this */
- public Optional<IssueId> ownershipIssueId() {
- return ownershipIssueId;
- }
-
- public Optional<User> owner() {
- return owner;
- }
-
- /**
- * Overrides the system major version for this application. This override takes effect if the deployment
- * spec does not specify a major version.
- */
- public OptionalInt majorVersion() { return majorVersion; }
-
- /** Returns metrics for this */
- public ApplicationMetrics metrics() {
- return metrics;
- }
-
/** Returns all rotations assigned to this */
public List<AssignedRotation> rotations() {
return rotations;
@@ -311,8 +188,6 @@ public class Instance {
return EndpointList.of(endpointStream);
}
- public Optional<String> pemDeployKey() { return pemDeployKey; }
-
/** Returns the status of the global rotation(s) assigned to this */
public RotationStatus rotationStatus() {
return rotationStatus;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index b8912a848fc..57a9c429ef7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -106,6 +106,14 @@ public class LockedApplication {
projectId, internal, instances.values());
}
+ public LockedApplication withNewInstance(InstanceName instance) {
+ var instances = new HashMap<>(this.instances);
+ instances.put(instance, new Instance(id.instance(instance)));
+ return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, change, outstandingChange,
+ deploymentIssueId, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
+ projectId, internal, instances);
+ }
+
public LockedApplication with(InstanceName instance, UnaryOperator<Instance> modification) {
var instances = new HashMap<>(this.instances);
instances.put(instance, modification.apply(instances.get(instance)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index 066ba0fbda5..0faf3d0301d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -5,9 +5,8 @@ import com.google.common.collect.ImmutableMap;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
import java.util.Collection;
import java.util.HashMap;
@@ -16,6 +15,10 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
+import java.util.function.Function;
+import java.util.function.UnaryOperator;
+
+import static java.util.stream.Collectors.toMap;
/**
* Information about which deployment jobs an application should run and their current status.
@@ -26,78 +29,39 @@ import java.util.OptionalLong;
*/
public class DeploymentJobs {
- private final OptionalLong projectId;
private final ImmutableMap<JobType, JobStatus> status;
- private final Optional<IssueId> issueId;
- private final boolean builtInternally;
-
- public DeploymentJobs(OptionalLong projectId, Collection<JobStatus> jobStatusEntries,
- Optional<IssueId> issueId, boolean builtInternally) {
- this(projectId, asMap(jobStatusEntries), issueId, builtInternally);
- }
-
- private DeploymentJobs(OptionalLong projectId, Map<JobType, JobStatus> status, Optional<IssueId> issueId,
- boolean builtInternally) {
- requireId(projectId, "projectId must be a positive integer");
- Objects.requireNonNull(status, "status cannot be null");
- Objects.requireNonNull(issueId, "issueId cannot be null");
- this.projectId = projectId;
- this.status = ImmutableMap.copyOf(status);
- this.issueId = issueId;
- this.builtInternally = builtInternally;
- }
- private static Map<JobType, JobStatus> asMap(Collection<JobStatus> jobStatusEntries) {
- ImmutableMap.Builder<JobType, JobStatus> b = new ImmutableMap.Builder<>();
- for (JobStatus jobStatusEntry : jobStatusEntries)
- b.put(jobStatusEntry.type(), jobStatusEntry);
- return b.build();
+ public DeploymentJobs(Collection<JobStatus> jobStatusEntries) {
+ this.status = ImmutableMap.copyOf(jobStatusEntries.stream().collect(toMap(JobStatus::type, Function.identity())));
}
- /** Return a new instance with the given completion */
- public DeploymentJobs withCompletion(JobType jobType, JobStatus.JobRun completion, Optional<JobError> jobError) {
+ /** Return a new instance with the given job update applied. */
+ public DeploymentJobs withUpdate(JobType jobType, UnaryOperator<JobStatus> update) {
Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status);
status.compute(jobType, (type, job) -> {
if (job == null) job = JobStatus.initial(jobType);
- return job.withCompletion(completion, jobError);
- });
- return new DeploymentJobs(projectId, status, issueId, builtInternally);
- }
-
- public DeploymentJobs withTriggering(JobType jobType, JobStatus.JobRun jobRun) {
- Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status);
- status.compute(jobType, (__, job) -> {
- if (job == null) job = JobStatus.initial(jobType);
- return job.withTriggering(jobRun);
+ return update.apply(job);
});
- return new DeploymentJobs(projectId, status, issueId, builtInternally);
+ return new DeploymentJobs(status.values());
}
- public DeploymentJobs withPause(JobType jobType, OptionalLong pausedUntil) {
- Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status);
- status.compute(jobType, (__, job) -> {
- if (job == null) job = JobStatus.initial(jobType);
- return job.withPause(pausedUntil);
- });
- return new DeploymentJobs(projectId, status, issueId, builtInternally);
+ /** Return a new instance with the given completion */
+ public DeploymentJobs withCompletion(JobType jobType, JobStatus.JobRun completion, Optional<JobError> jobError) {
+ return withUpdate(jobType, job -> job.withCompletion(completion, jobError));
}
- public DeploymentJobs withProjectId(OptionalLong projectId) {
- return new DeploymentJobs(projectId, status, issueId, builtInternally);
+ public DeploymentJobs withTriggering(JobType jobType, JobStatus.JobRun jobRun) {
+ return withUpdate(jobType, job -> job.withTriggering(jobRun));
}
- public DeploymentJobs with(IssueId issueId) {
- return new DeploymentJobs(projectId, status, Optional.ofNullable(issueId), builtInternally);
+ public DeploymentJobs withPause(JobType jobType, OptionalLong pausedUntil) {
+ return withUpdate(jobType, job -> job.withPause(pausedUntil));
}
public DeploymentJobs without(JobType job) {
Map<JobType, JobStatus> status = new HashMap<>(this.status);
status.remove(job);
- return new DeploymentJobs(projectId, status, issueId, builtInternally);
- }
-
- public DeploymentJobs withBuiltInternally(boolean builtInternally) {
- return new DeploymentJobs(projectId, status, issueId, builtInternally);
+ return new DeploymentJobs(status.values());
}
/** Returns an immutable map of the status entries in this */
@@ -116,28 +80,6 @@ public class DeploymentJobs {
return Optional.ofNullable(jobStatus().get(jobType));
}
- /**
- * Returns the id of the Screwdriver project running these deployment jobs
- * - or empty when this is not known or does not exist.
- * It is not known until the jobs have run once and reported back to the controller.
- */
- public OptionalLong projectId() { return projectId; }
-
- public Optional<IssueId> issueId() { return issueId; }
-
- public boolean deployedInternally() { return builtInternally; }
-
- private static OptionalLong requireId(OptionalLong id, String message) {
- Objects.requireNonNull(id, message);
- if ( ! id.isPresent()) {
- return id;
- }
- if (id.getAsLong() <= 0) {
- throw new IllegalArgumentException(message);
- }
- return id;
- }
-
/** A job report. This class is immutable. */
public static class JobReport {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 1a2524fa05e..47c59e1d425 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -539,8 +539,7 @@ public class ApplicationSerializer {
private DeploymentJobs deploymentJobsFromSlime(Inspector object) {
List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField));
-
- return new DeploymentJobs(OptionalLong.empty(), jobStatusList, Optional.empty(), false); // WARNING: Unused variables.
+ return new DeploymentJobs(jobStatusList);
}
private Change changeFromSlime(Inspector object) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 2503aa1dd6d..df0fc4b4f36 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -87,7 +87,6 @@ public class CuratorDb {
private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer();
private final TenantSerializer tenantSerializer = new TenantSerializer();
private final ApplicationSerializer applicationSerializer = new ApplicationSerializer();
- private final InstanceSerializer instanceSerializer = new InstanceSerializer();
private final RunSerializer runSerializer = new RunSerializer();
private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer();
private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer);
@@ -337,10 +336,6 @@ public class CuratorDb {
public void writeApplication(Application application) {
curator.set(applicationPath(application.id()), asJson(applicationSerializer.toSlime(application)));
- for (InstanceName name : application.instances().keySet()) {
- curator.set(oldApplicationPath(application.id().instance(name)),
- asJson(instanceSerializer.toSlime(application.legacy(name))));
- }
}
public Optional<Application> readApplication(TenantAndApplicationId application) {
@@ -376,8 +371,7 @@ public class CuratorDb {
}
// TODO jonmv: Refactor when instance split operation is done
- public void storeWithoutInstance(Application application, ApplicationId instanceId) {
- curator.delete(oldApplicationPath(instanceId));
+ public void storeWithoutInstance(Application application) {
if (application.instances().isEmpty())
curator.delete(applicationPath(application.id()));
else
@@ -390,7 +384,7 @@ public class CuratorDb {
* Add filter for reading only Instance from old application path RELEASED
* Write Instance to instance and old application path RELEASED
*
- * Lock on application level for instance mutations MERGED
+ * Lock on application level for instance mutations RELEASED
*
* Write Instance to instance and application and old application paths DONE TO CHANGE DONE
* Read Instance from instance path DONE TO REMOVE DONE
@@ -407,7 +401,7 @@ public class CuratorDb {
*
* Read Application with instances from application path (with filter) DONE
*
- * Stop writing Instance to old application path
+ * Stop writing Instance to old application path DONE
* Remove unused parts of Instance (Used only for legacy serialization)
* Store new production application packages under non-instance path
* Read production packages from non-instance path, with fallback
@@ -636,10 +630,6 @@ public class CuratorDb {
return applicationRoot.append(id.serialized());
}
- private static Path oldApplicationPath(ApplicationId application) {
- return applicationRoot.append(application.serializedForm());
- }
-
private static Path runsPath(ApplicationId id, JobType type) {
return jobRoot.append(id.serializedForm()).append(type.jobName());
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializer.java
deleted file mode 100644
index 3cd594a277d..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializer.java
+++ /dev/null
@@ -1,574 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.persistence;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.application.api.ValidationOverrides;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.slime.ArrayTraverser;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Inspector;
-import com.yahoo.slime.ObjectTraverser;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
-import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
-import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
-import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
-
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.OptionalInt;
-import java.util.OptionalLong;
-import java.util.stream.Collectors;
-
-/**
- * Serializes {@link Instance} to/from slime.
- * This class is multithread safe.
- *
- * @author bratseth
- */
-public class InstanceSerializer {
-
- // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one
- // (and rewrite all nodes on startup), changes to the serialized format must be made
- // such that what is serialized on version N+1 can be read by version N:
- // - ADDING FIELDS: Always ok
- // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version.
- // - CHANGING THE FORMAT OF A FIELD: Don't do it bro.
-
- // Application fields
- private static final String idField = "id";
- private static final String createdAtField = "createdAt";
- private static final String deploymentSpecField = "deploymentSpecField";
- private static final String validationOverridesField = "validationOverrides";
- private static final String deploymentsField = "deployments";
- private static final String deploymentJobsField = "deploymentJobs";
- private static final String deployingField = "deployingField";
- private static final String pinnedField = "pinned";
- private static final String outstandingChangeField = "outstandingChangeField";
- private static final String ownershipIssueIdField = "ownershipIssueId";
- private static final String ownerField = "confirmedOwner";
- private static final String majorVersionField = "majorVersion";
- private static final String writeQualityField = "writeQuality";
- private static final String queryQualityField = "queryQuality";
- private static final String pemDeployKeyField = "pemDeployKey";
- private static final String assignedRotationsField = "assignedRotations";
- private static final String assignedRotationEndpointField = "endpointId";
- private static final String assignedRotationClusterField = "clusterId";
- private static final String assignedRotationRotationField = "rotationId";
- private static final String applicationCertificateField = "applicationCertificate";
-
- // Deployment fields
- private static final String zoneField = "zone";
- private static final String environmentField = "environment";
- private static final String regionField = "region";
- private static final String deployTimeField = "deployTime";
- private static final String applicationBuildNumberField = "applicationBuildNumber";
- private static final String applicationPackageRevisionField = "applicationPackageRevision";
- private static final String sourceRevisionField = "sourceRevision";
- private static final String repositoryField = "repositoryField";
- private static final String branchField = "branchField";
- private static final String commitField = "commitField";
- private static final String authorEmailField = "authorEmailField";
- private static final String compileVersionField = "compileVersion";
- private static final String buildTimeField = "buildTime";
- private static final String lastQueriedField = "lastQueried";
- private static final String lastWrittenField = "lastWritten";
- private static final String lastQueriesPerSecondField = "lastQueriesPerSecond";
- private static final String lastWritesPerSecondField = "lastWritesPerSecond";
-
- // DeploymentJobs fields
- private static final String projectIdField = "projectId";
- private static final String jobStatusField = "jobStatus";
- private static final String issueIdField = "jiraIssueId";
- private static final String builtInternallyField = "builtInternally";
-
- // JobStatus field
- private static final String jobTypeField = "jobType";
- private static final String errorField = "jobError";
- private static final String lastTriggeredField = "lastTriggered";
- private static final String lastCompletedField = "lastCompleted";
- private static final String firstFailingField = "firstFailing";
- private static final String lastSuccessField = "lastSuccess";
- private static final String pausedUntilField = "pausedUntil";
-
- // JobRun fields
- private static final String jobRunIdField = "id";
- private static final String versionField = "version";
- private static final String revisionField = "revision";
- private static final String sourceVersionField = "sourceVersion";
- private static final String sourceApplicationField = "sourceRevision";
- private static final String reasonField = "reason";
- private static final String atField = "at";
-
- // ClusterInfo fields
- private static final String clusterInfoField = "clusterInfo";
- private static final String clusterInfoFlavorField = "flavor";
- private static final String clusterInfoCostField = "cost";
- private static final String clusterInfoCpuField = "flavorCpu";
- private static final String clusterInfoMemField = "flavorMem";
- private static final String clusterInfoDiskField = "flavorDisk";
- private static final String clusterInfoTypeField = "clusterType";
- private static final String clusterInfoHostnamesField = "hostnames";
-
- // ClusterUtils fields
- private static final String clusterUtilsField = "clusterUtils";
- private static final String clusterUtilsCpuField = "cpu";
- private static final String clusterUtilsMemField = "mem";
- private static final String clusterUtilsDiskField = "disk";
- private static final String clusterUtilsDiskBusyField = "diskbusy";
-
- // Deployment metrics fields
- private static final String deploymentMetricsField = "metrics";
- private static final String deploymentMetricsQPSField = "queriesPerSecond";
- private static final String deploymentMetricsWPSField = "writesPerSecond";
- private static final String deploymentMetricsDocsField = "documentCount";
- private static final String deploymentMetricsQueryLatencyField = "queryLatencyMillis";
- private static final String deploymentMetricsWriteLatencyField = "writeLatencyMillis";
- private static final String deploymentMetricsUpdateTime = "lastUpdated";
- private static final String deploymentMetricsWarningsField = "warnings";
-
- // RotationStatus fields
- private static final String rotationStatusField = "rotationStatus2";
- private static final String rotationIdField = "rotationId";
- private static final String rotationStateField = "state";
- private static final String statusField = "status";
-
- // ------------------ Serialization
-
- public Slime toSlime(Instance instance) {
- Slime slime = new Slime();
- Cursor root = slime.setObject();
- root.setString(idField, instance.id().serializedForm());
- root.setLong(createdAtField, instance.createdAt().toEpochMilli());
- root.setString(deploymentSpecField, instance.deploymentSpec().xmlForm());
- root.setString(validationOverridesField, instance.validationOverrides().xmlForm());
- deploymentsToSlime(instance.deployments().values(), root.setArray(deploymentsField));
- toSlime(instance.deploymentJobs(), root.setObject(deploymentJobsField));
- toSlime(instance.change(), root, deployingField);
- toSlime(instance.outstandingChange(), root, outstandingChangeField);
- instance.ownershipIssueId().ifPresent(issueId -> root.setString(ownershipIssueIdField, issueId.value()));
- instance.owner().ifPresent(owner -> root.setString(ownerField, owner.username()));
- instance.majorVersion().ifPresent(majorVersion -> root.setLong(majorVersionField, majorVersion));
- root.setDouble(queryQualityField, instance.metrics().queryServiceQuality());
- root.setDouble(writeQualityField, instance.metrics().writeServiceQuality());
- instance.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey));
- assignedRotationsToSlime(instance.rotations(), root, assignedRotationsField);
- toSlime(instance.rotationStatus(), root.setArray(rotationStatusField));
- return slime;
- }
-
- private void deploymentsToSlime(Collection<Deployment> deployments, Cursor array) {
- for (Deployment deployment : deployments)
- deploymentToSlime(deployment, array.addObject());
- }
-
- private void deploymentToSlime(Deployment deployment, Cursor object) {
- zoneIdToSlime(deployment.zone(), object.setObject(zoneField));
- object.setString(versionField, deployment.version().toString());
- object.setLong(deployTimeField, deployment.at().toEpochMilli());
- toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField));
- clusterInfoToSlime(deployment.clusterInfo(), object);
- clusterUtilsToSlime(deployment.clusterUtils(), object);
- deploymentMetricsToSlime(deployment.metrics(), object);
- deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli()));
- deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli()));
- deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value));
- deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value));
- }
-
- private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) {
- Cursor root = object.setObject(deploymentMetricsField);
- root.setDouble(deploymentMetricsQPSField, metrics.queriesPerSecond());
- root.setDouble(deploymentMetricsWPSField, metrics.writesPerSecond());
- root.setDouble(deploymentMetricsDocsField, metrics.documentCount());
- root.setDouble(deploymentMetricsQueryLatencyField, metrics.queryLatencyMillis());
- root.setDouble(deploymentMetricsWriteLatencyField, metrics.writeLatencyMillis());
- metrics.instant().ifPresent(instant -> root.setLong(deploymentMetricsUpdateTime, instant.toEpochMilli()));
- if (!metrics.warnings().isEmpty()) {
- Cursor warningsObject = root.setObject(deploymentMetricsWarningsField);
- metrics.warnings().forEach((warning, count) -> warningsObject.setLong(warning.name(), count));
- }
- }
-
- private void clusterInfoToSlime(Map<ClusterSpec.Id, ClusterInfo> clusters, Cursor object) {
- Cursor root = object.setObject(clusterInfoField);
- for (Map.Entry<ClusterSpec.Id, ClusterInfo> entry : clusters.entrySet()) {
- toSlime(entry.getValue(), root.setObject(entry.getKey().value()));
- }
- }
-
- private void toSlime(ClusterInfo info, Cursor object) {
- object.setString(clusterInfoFlavorField, info.getFlavor());
- object.setLong(clusterInfoCostField, info.getFlavorCost());
- object.setDouble(clusterInfoCpuField, info.getFlavorCPU());
- object.setDouble(clusterInfoMemField, info.getFlavorMem());
- object.setDouble(clusterInfoDiskField, info.getFlavorDisk());
- object.setString(clusterInfoTypeField, info.getClusterType().name());
- Cursor array = object.setArray(clusterInfoHostnamesField);
- for (String host : info.getHostnames()) {
- array.addString(host);
- }
- }
-
- private void clusterUtilsToSlime(Map<ClusterSpec.Id, ClusterUtilization> clusters, Cursor object) {
- Cursor root = object.setObject(clusterUtilsField);
- for (Map.Entry<ClusterSpec.Id, ClusterUtilization> entry : clusters.entrySet()) {
- toSlime(entry.getValue(), root.setObject(entry.getKey().value()));
- }
- }
-
- private void toSlime(ClusterUtilization utils, Cursor object) {
- object.setDouble(clusterUtilsCpuField, utils.getCpu());
- object.setDouble(clusterUtilsMemField, utils.getMemory());
- object.setDouble(clusterUtilsDiskField, utils.getDisk());
- object.setDouble(clusterUtilsDiskBusyField, utils.getDiskBusy());
- }
-
- private void zoneIdToSlime(ZoneId zone, Cursor object) {
- object.setString(environmentField, zone.environment().value());
- object.setString(regionField, zone.region().value());
- }
-
- private void toSlime(ApplicationVersion applicationVersion, Cursor object) {
- if (applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent()) {
- object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong());
- toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField));
- applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email));
- applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString()));
- applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli()));
- }
- }
-
- private void toSlime(SourceRevision sourceRevision, Cursor object) {
- object.setString(repositoryField, sourceRevision.repository());
- object.setString(branchField, sourceRevision.branch());
- object.setString(commitField, sourceRevision.commit());
- }
-
- private void toSlime(DeploymentJobs deploymentJobs, Cursor cursor) {
- deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId));
- jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField));
- deploymentJobs.issueId().ifPresent(jiraIssueId -> cursor.setString(issueIdField, jiraIssueId.value()));
- cursor.setBool(builtInternallyField, deploymentJobs.deployedInternally());
- }
-
- private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) {
- for (JobStatus jobStatus : jobStatuses)
- toSlime(jobStatus, jobStatusArray.addObject());
- }
-
- private void toSlime(JobStatus jobStatus, Cursor object) {
- object.setString(jobTypeField, jobStatus.type().jobName());
- if (jobStatus.jobError().isPresent())
- object.setString(errorField, jobStatus.jobError().get().name());
-
- jobStatus.lastTriggered().ifPresent(run -> jobRunToSlime(run, object, lastTriggeredField));
- jobStatus.lastCompleted().ifPresent(run -> jobRunToSlime(run, object, lastCompletedField));
- jobStatus.lastSuccess().ifPresent(run -> jobRunToSlime(run, object, lastSuccessField));
- jobStatus.firstFailing().ifPresent(run -> jobRunToSlime(run, object, firstFailingField));
- jobStatus.pausedUntil().ifPresent(until -> object.setLong(pausedUntilField, until));
- }
-
- private void jobRunToSlime(JobStatus.JobRun jobRun, Cursor parent, String jobRunObjectName) {
- Cursor object = parent.setObject(jobRunObjectName);
- object.setLong(jobRunIdField, jobRun.id());
- object.setString(versionField, jobRun.platform().toString());
- toSlime(jobRun.application(), object.setObject(revisionField));
- jobRun.sourcePlatform().ifPresent(version -> object.setString(sourceVersionField, version.toString()));
- jobRun.sourceApplication().ifPresent(version -> toSlime(version, object.setObject(sourceApplicationField)));
- object.setString(reasonField, jobRun.reason());
- object.setLong(atField, jobRun.at().toEpochMilli());
- }
-
- private void toSlime(Change deploying, Cursor parentObject, String fieldName) {
- if (deploying.isEmpty()) return;
-
- Cursor object = parentObject.setObject(fieldName);
- if (deploying.platform().isPresent())
- object.setString(versionField, deploying.platform().get().toString());
- if (deploying.application().isPresent())
- toSlime(deploying.application().get(), object);
- if (deploying.isPinned())
- object.setBool(pinnedField, true);
- }
-
- private void toSlime(RotationStatus status, Cursor array) {
- status.asMap().forEach((rotationId, zoneStatus) -> {
- Cursor rotationObject = array.addObject();
- rotationObject.setString(rotationIdField, rotationId.asString());
- Cursor statusArray = rotationObject.setArray(statusField);
- zoneStatus.forEach((zone, state) -> {
- Cursor statusObject = statusArray.addObject();
- zoneIdToSlime(zone, statusObject);
- statusObject.setString(rotationStateField, state.name());
- });
- });
- }
-
- private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) {
- var rotationsArray = parent.setArray(fieldName);
- for (var rotation : rotations) {
- var object = rotationsArray.addObject();
- object.setString(assignedRotationEndpointField, rotation.endpointId().id());
- object.setString(assignedRotationRotationField, rotation.rotationId().asString());
- object.setString(assignedRotationClusterField, rotation.clusterId().value());
- }
- }
-
- // ------------------ Deserialization
-
- public Instance fromSlime(Slime slime) {
- Inspector root = slime.get();
-
- ApplicationId id = ApplicationId.fromSerializedForm(root.field(idField).asString());
- Instant createdAt = Instant.ofEpochMilli(root.field(createdAtField).asLong());
- DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false);
- ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString());
- List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField));
- DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField));
- Change deploying = changeFromSlime(root.field(deployingField));
- Change outstandingChange = changeFromSlime(root.field(outstandingChangeField));
- Optional<IssueId> ownershipIssueId = Serializers.optionalString(root.field(ownershipIssueIdField)).map(IssueId::from);
- Optional<User> owner = Serializers.optionalString(root.field(ownerField)).map(User::from);
- OptionalInt majorVersion = Serializers.optionalInteger(root.field(majorVersionField));
- ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
- root.field(writeQualityField).asDouble());
- Optional<String> pemDeployKey = Serializers.optionalString(root.field(pemDeployKeyField));
- List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(deploymentSpec, root);
- RotationStatus rotationStatus = rotationStatusFromSlime(root);
-
- return new Instance(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs,
- deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics,
- pemDeployKey, assignedRotations, rotationStatus);
- }
-
- private List<Deployment> deploymentsFromSlime(Inspector array) {
- List<Deployment> deployments = new ArrayList<>();
- array.traverse((ArrayTraverser) (int i, Inspector item) -> deployments.add(deploymentFromSlime(item)));
- return deployments;
- }
-
- private Deployment deploymentFromSlime(Inspector deploymentObject) {
- return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)),
- applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)),
- Version.fromString(deploymentObject.field(versionField).asString()),
- Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()),
- clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)),
- clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)),
- deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)),
- DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)),
- Serializers.optionalInstant(deploymentObject.field(lastWrittenField)),
- Serializers.optionalDouble(deploymentObject.field(lastQueriesPerSecondField)),
- Serializers.optionalDouble(deploymentObject.field(lastWritesPerSecondField))));
- }
-
- private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) {
- Optional<Instant> instant = object.field(deploymentMetricsUpdateTime).valid() ?
- Optional.of(Instant.ofEpochMilli(object.field(deploymentMetricsUpdateTime).asLong())) :
- Optional.empty();
- return new DeploymentMetrics(object.field(deploymentMetricsQPSField).asDouble(),
- object.field(deploymentMetricsWPSField).asDouble(),
- object.field(deploymentMetricsDocsField).asDouble(),
- object.field(deploymentMetricsQueryLatencyField).asDouble(),
- object.field(deploymentMetricsWriteLatencyField).asDouble(),
- instant,
- deploymentWarningsFrom(object.field(deploymentMetricsWarningsField)));
- }
-
- private Map<DeploymentMetrics.Warning, Integer> deploymentWarningsFrom(Inspector object) {
- Map<DeploymentMetrics.Warning, Integer> warnings = new HashMap<>();
- object.traverse((ObjectTraverser) (name, value) -> warnings.put(DeploymentMetrics.Warning.valueOf(name),
- (int) value.asLong()));
- return Collections.unmodifiableMap(warnings);
- }
-
- private RotationStatus rotationStatusFromSlime(Inspector parentObject) {
- var object = parentObject.field(rotationStatusField);
- var statusMap = new LinkedHashMap<RotationId, Map<ZoneId, RotationState>>();
- object.traverse((ArrayTraverser) (idx, statusObject) -> statusMap.put(new RotationId(statusObject.field(rotationIdField).asString()),
- singleRotationStatusFromSlime(statusObject.field(statusField))));
- return RotationStatus.from(statusMap);
- }
-
- private Map<ZoneId, RotationState> singleRotationStatusFromSlime(Inspector object) {
- if (!object.valid()) {
- return Collections.emptyMap();
- }
- Map<ZoneId, RotationState> rotationStatus = new LinkedHashMap<>();
- object.traverse((ArrayTraverser) (idx, statusObject) -> {
- var zone = zoneIdFromSlime(statusObject);
- var status = RotationState.valueOf(statusObject.field(rotationStateField).asString());
- rotationStatus.put(zone, status);
- });
- return Collections.unmodifiableMap(rotationStatus);
- }
-
- private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime (Inspector object) {
- Map<ClusterSpec.Id, ClusterInfo> map = new HashMap<>();
- object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterInfoFromSlime(value)));
- return map;
- }
-
- private Map<ClusterSpec.Id, ClusterUtilization> clusterUtilsMapFromSlime(Inspector object) {
- Map<ClusterSpec.Id, ClusterUtilization> map = new HashMap<>();
- object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterUtililzationFromSlime(value)));
- return map;
- }
-
- private ClusterUtilization clusterUtililzationFromSlime(Inspector object) {
- double cpu = object.field(clusterUtilsCpuField).asDouble();
- double mem = object.field(clusterUtilsMemField).asDouble();
- double disk = object.field(clusterUtilsDiskField).asDouble();
- double diskBusy = object.field(clusterUtilsDiskBusyField).asDouble();
-
- return new ClusterUtilization(mem, cpu, disk, diskBusy);
- }
-
- private ClusterInfo clusterInfoFromSlime(Inspector inspector) {
- String flavor = inspector.field(clusterInfoFlavorField).asString();
- int cost = (int)inspector.field(clusterInfoCostField).asLong();
- String type = inspector.field(clusterInfoTypeField).asString();
- double flavorCpu = inspector.field(clusterInfoCpuField).asDouble();
- double flavorMem = inspector.field(clusterInfoMemField).asDouble();
- double flavorDisk = inspector.field(clusterInfoDiskField).asDouble();
-
- List<String> hostnames = new ArrayList<>();
- inspector.field(clusterInfoHostnamesField).traverse((ArrayTraverser)(int index, Inspector value) -> hostnames.add(value.asString()));
- return new ClusterInfo(flavor, cost, flavorCpu, flavorMem, flavorDisk, ClusterSpec.Type.from(type), hostnames);
- }
-
- private ZoneId zoneIdFromSlime(Inspector object) {
- return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString());
- }
-
- private ApplicationVersion applicationVersionFromSlime(Inspector object) {
- if ( ! object.valid()) return ApplicationVersion.unknown;
- OptionalLong applicationBuildNumber = Serializers.optionalLong(object.field(applicationBuildNumberField));
- Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField));
- if (sourceRevision.isEmpty() || applicationBuildNumber.isEmpty()) {
- return ApplicationVersion.unknown;
- }
- Optional<String> authorEmail = Serializers.optionalString(object.field(authorEmailField));
- Optional<Version> compileVersion = Serializers.optionalString(object.field(compileVersionField)).map(Version::fromString);
- Optional<Instant> buildTime = Serializers.optionalInstant(object.field(buildTimeField));
-
- if (authorEmail.isEmpty())
- return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong());
-
- if (compileVersion.isEmpty() || buildTime.isEmpty())
- return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get());
-
- return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get(),
- compileVersion.get(), buildTime.get());
- }
-
- private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) {
- if ( ! object.valid()) return Optional.empty();
- return Optional.of(new SourceRevision(object.field(repositoryField).asString(),
- object.field(branchField).asString(),
- object.field(commitField).asString()));
- }
-
- private DeploymentJobs deploymentJobsFromSlime(Inspector object) {
- OptionalLong projectId = Serializers.optionalLong(object.field(projectIdField));
- List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField));
- Optional<IssueId> issueId = Serializers.optionalString(object.field(issueIdField)).map(IssueId::from);
- boolean builtInternally = object.field(builtInternallyField).asBool();
-
- return new DeploymentJobs(projectId, jobStatusList, issueId, builtInternally);
- }
-
- private Change changeFromSlime(Inspector object) {
- if ( ! object.valid()) return Change.empty();
- Inspector versionFieldValue = object.field(versionField);
- Change change = Change.empty();
- if (versionFieldValue.valid())
- change = Change.of(Version.fromString(versionFieldValue.asString()));
- if (object.field(applicationBuildNumberField).valid())
- change = change.with(applicationVersionFromSlime(object));
- if (object.field(pinnedField).asBool())
- change = change.withPin();
- return change;
- }
-
- private List<JobStatus> jobStatusListFromSlime(Inspector array) {
- List<JobStatus> jobStatusList = new ArrayList<>();
- array.traverse((ArrayTraverser) (int i, Inspector item) -> jobStatusFromSlime(item).ifPresent(jobStatusList::add));
- return jobStatusList;
- }
-
- private Optional<JobStatus> jobStatusFromSlime(Inspector object) {
- // if the job type has since been removed, ignore it
- Optional<JobType> jobType =
- JobType.fromOptionalJobName(object.field(jobTypeField).asString());
- if (jobType.isEmpty()) return Optional.empty();
-
- Optional<JobError> jobError = Optional.empty();
- if (object.field(errorField).valid())
- jobError = Optional.of(JobError.valueOf(object.field(errorField).asString()));
-
- return Optional.of(new JobStatus(jobType.get(),
- jobError,
- jobRunFromSlime(object.field(lastTriggeredField)),
- jobRunFromSlime(object.field(lastCompletedField)),
- jobRunFromSlime(object.field(firstFailingField)),
- jobRunFromSlime(object.field(lastSuccessField)),
- Serializers.optionalLong(object.field(pausedUntilField))));
- }
-
- private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) {
- if ( ! object.valid()) return Optional.empty();
- return Optional.of(new JobStatus.JobRun(object.field(jobRunIdField).asLong(),
- new Version(object.field(versionField).asString()),
- applicationVersionFromSlime(object.field(revisionField)),
- Serializers.optionalString(object.field(sourceVersionField)).map(Version::fromString),
- Optional.of(object.field(sourceApplicationField)).filter(Inspector::valid).map(this::applicationVersionFromSlime),
- object.field(reasonField).asString(),
- Instant.ofEpochMilli(object.field(atField).asLong())));
- }
-
- private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) {
- var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>();
-
- root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> {
- var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString());
- var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString());
- var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString());
- var regions = deploymentSpec.endpoints().stream()
- .filter(endpoint -> endpoint.endpointId().equals(endpointId.id()))
- .flatMap(endpoint -> endpoint.regions().stream())
- .collect(Collectors.toSet());
- assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions));
- });
-
- return List.copyOf(assignedRotations.values());
- }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 90d257d754d..a4f3a804c55 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -7,16 +7,12 @@ import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.AthenzDomain;
-import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.slime.JsonFormat;
import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
@@ -24,17 +20,12 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
-import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.JobStatus;
@@ -43,25 +34,15 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
-import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
-import com.yahoo.vespa.hosted.controller.persistence.InstanceSerializer;
-import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
-import com.yahoo.vespa.hosted.controller.persistence.OldMockCuratorDb;
import com.yahoo.vespa.hosted.controller.rotation.RotationId;
import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
import org.junit.Test;
-import java.io.IOException;
import java.time.Duration;
-import java.time.Instant;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.OptionalInt;
-import java.util.OptionalLong;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
@@ -72,7 +53,6 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
-import static java.nio.charset.StandardCharsets.UTF_8;
import static java.time.temporal.ChronoUnit.MILLIS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -790,103 +770,6 @@ public class ControllerTest {
}
}
-
- @Test
- public void testInstanceDataMigration() throws IOException {
- /*
- Set up initial state, using old DB:
- Create two instances of an application; the default will be the base.
-
- Read, modify and write the application using the new DB.
- Verify results using both old and new DBs.
- */
-
- ApplicationPackage applicationPackage = new ApplicationPackageBuilder().allow(ValidationId.contentClusterRemoval)
- .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
- .endpoint("endpoint", "container", "us-east-1")
- .region("us-east-1")
- .build();
- ApplicationId defaultId = ApplicationId.from("t1", "a1", "default");
- Instance old1 = new Instance(defaultId,
- Instant.ofEpochMilli(123),
- applicationPackage.deploymentSpec(),
- applicationPackage.validationOverrides(),
- List.of(new Deployment(ZoneId.from("prod", "us-east-1"),
- ApplicationVersion.from(new SourceRevision("repo", "branch", "commit"), 3),
- Version.fromString("7.8.9"),
- Instant.ofEpochMilli(321))),
- new DeploymentJobs(OptionalLong.of(72),
- List.of(new JobStatus(JobType.productionAwsUsEast1a,
- Optional.empty(),
- Optional.of(new JobStatus.JobRun(32,
- Version.fromString("7.8.9"),
- ApplicationVersion.unknown,
- Optional.empty(),
- Optional.empty(),
- "make the job great again",
- Instant.ofEpochMilli(200))),
- Optional.empty(),
- Optional.empty(),
- Optional.empty(),
- OptionalLong.empty())),
- Optional.of(IssueId.from("issue")),
- true),
- Change.of(Version.fromString("9")),
- Change.empty(),
- Optional.of(IssueId.from("tissue")),
- Optional.of(User.from("user")),
- OptionalInt.of(3),
- new ApplicationMetrics(2, 3),
- Optional.of("key"),
- List.of(AssignedRotation.fromStrings("container", "endpoint", "rot13", List.of("us-east-1"))),
- RotationStatus.from(Map.of(new RotationId("rot13"), Map.of(ZoneId.from("prod", "us-east-1"), RotationState.in))));
-
- Instance old2 = new Instance(ApplicationId.from("t1", "a1", "i1"),
- Instant.ofEpochMilli(400));
-
-
- InstanceSerializer instanceSerializer = new InstanceSerializer();
- String old1Serialized = new String(JsonFormat.toJsonBytes(instanceSerializer.toSlime(old1)), UTF_8);
-
- MockCuratorDb newDb = new MockCuratorDb();
- OldMockCuratorDb oldDb = new OldMockCuratorDb(newDb.curator());
-
- oldDb.writeApplication(Application.aggregate(List.of(old1, old2)).orElseThrow());
-
- Application application = oldDb.readApplication(TenantAndApplicationId.from("t1", "a1")).orElseThrow();
- Instance new1 = application.legacy(InstanceName.defaultName());
- String new1Serialized = new String(JsonFormat.toJsonBytes(instanceSerializer.toSlime(new1)), UTF_8);
- assertEquals(old1Serialized, new1Serialized);
-
- LockedApplication locked = new LockedApplication(application, newDb.lock(application.id()));
- oldDb.writeApplication(locked.with(new ApplicationMetrics(8, 9)).get());
-
- Application newApp = newDb.readApplication(application.id()).orElseThrow();
- Instance mod1 = newApp.legacy(old1.name());
- Instance mod2 = newApp.legacy(old2.name());
-
- old1 = old1.with(new ApplicationMetrics(8, 9));
- old1Serialized = new String(JsonFormat.toJsonBytes(instanceSerializer.toSlime(old1)), UTF_8);
- String mod1Serialized = new String(JsonFormat.toJsonBytes(instanceSerializer.toSlime(mod1)), UTF_8);
- assertEquals(old1Serialized, mod1Serialized);
-
- assertEquals(old1.createdAt(), mod2.createdAt());
- assertEquals(old1.change(), mod2.change());
- assertEquals(old1.outstandingChange(), mod2.outstandingChange());
- assertEquals(old1.deploymentSpec(), mod2.deploymentSpec());
- assertEquals(old2.deployments(), mod2.deployments());
- assertEquals(old2.deploymentJobs().jobStatus(), mod2.deploymentJobs().jobStatus());
-
- application = new LockedApplication(application, newDb.lock(application.id())).without(old1.name()).get();
- newDb.storeWithoutInstance(application, old1.id());
- assertEquals(1, newDb.readApplication(application.id()).orElseThrow().instances().size());
- assertEquals(1, oldDb.readApplication(application.id()).orElseThrow().instances().size());
- application = new LockedApplication(application, newDb.lock(application.id())).without(old2.name()).get();
- newDb.storeWithoutInstance(application, old2.id());
- assertTrue(newDb.readApplication(application.id()).isEmpty());
- assertTrue(oldDb.readApplication(application.id()).isEmpty());
- }
-
private void runUpgrade(DeploymentTester tester, ApplicationId application, ApplicationVersion version) {
Version next = Version.fromString("6.2");
tester.upgradeSystem(next);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 82d9690f7d7..cefdc3bed61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -33,7 +33,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
-import com.yahoo.vespa.hosted.controller.persistence.InstanceSerializer;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
import com.yahoo.vespa.hosted.controller.security.AthenzCredentials;
@@ -182,15 +181,6 @@ public final class ControllerTester {
createAndDeploy(tenantName, domainName, applicationName, environment, projectId, null);
}
- /** Create application from slime */
- public void createApplication(Slime slime) {
- Instance instance = new InstanceSerializer().fromSlime(slime);
- Application application = Application.aggregate(List.of(instance)).get();
- try (Lock lock = controller().applications().lock(application.id())) {
- controller().applications().store(new LockedApplication(application, lock));
- }
- }
-
public ZoneId toZone(Environment environment) {
switch (environment) {
case dev: case test:
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 545729b5474..744ef0f3b08 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -36,6 +36,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Optional;
+import java.util.OptionalLong;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@@ -109,6 +110,12 @@ public class DeploymentTriggerTest {
tester.deployAndNotify(instance.id(), Optional.of(applicationPackage), true, JobType.systemTest);
tester.assertRunning(productionUsWest1, app.id().defaultInstance());
+
+ // system-test fails again, but the app loses its projectId, and the job isn't retried.
+ tester.applications().lockApplicationOrThrow(app.id(), locked ->
+ tester.applications().store(locked.withProjectId(OptionalLong.empty())));
+ tester.deployAndNotify(instance.id(), Optional.of(applicationPackage), false, productionUsWest1);
+ assertEquals("Job is not triggered when no projectId is present", 0, tester.buildService().jobs().size());
}
@Test
@@ -1061,23 +1068,6 @@ public class DeploymentTriggerTest {
}
@Test
- public void applicationWithoutProjectIdIsNotTriggered() throws Exception {
- // Current system version, matches version in test data
- Version version = Version.fromString("6.42.1");
- tester.upgradeSystem(version);
- assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
-
- // Load test data data
- byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json"));
- Slime slime = SlimeUtils.jsonToSlime(json);
- tester.controllerTester().createApplication(slime);
-
- // Failure redeployer does not restart deployment
- tester.readyJobTrigger().maintain();
- assertTrue("No jobs scheduled", tester.buildService().jobs().isEmpty());
- }
-
- @Test
public void testPlatformVersionSelection() {
// Setup system
ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 5889b71bdd9..3e8a66249bf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -101,7 +101,7 @@ public class ApplicationSerializerTest {
.withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, deployments.stream().findFirst(), "Test 3", Instant.ofEpochMilli(6))
.withCompletion(11, empty(), Instant.ofEpochMilli(7)));
- DeploymentJobs deploymentJobs = new DeploymentJobs(OptionalLong.empty(), statusList, empty(), true);
+ DeploymentJobs deploymentJobs = new DeploymentJobs(statusList);
var rotationStatus = RotationStatus.from(Map.of(new RotationId("my-rotation"),
Map.of(ZoneId.from("prod", "us-west-1"), RotationState.in,
@@ -116,7 +116,7 @@ public class ApplicationSerializerTest {
rotationStatus),
new Instance(id3,
List.of(),
- new DeploymentJobs(OptionalLong.empty(), List.of(), empty(), true),
+ new DeploymentJobs(List.of()),
List.of(),
RotationStatus.EMPTY));
@@ -163,7 +163,6 @@ public class ApplicationSerializerTest {
assertEquals(original.require(id1.instance()).deployments().get(zone2).activity().lastQueried().get(), serialized.require(id1.instance()).deployments().get(zone2).activity().lastQueried().get());
assertEquals(original.require(id1.instance()).deployments().get(zone2).activity().lastWritten().get(), serialized.require(id1.instance()).deployments().get(zone2).activity().lastWritten().get());
- assertEquals(original.require(id1.instance()).deploymentJobs().projectId(), serialized.require(id1.instance()).deploymentJobs().projectId());
assertEquals(original.require(id1.instance()).deploymentJobs().jobStatus().size(), serialized.require(id1.instance()).deploymentJobs().jobStatus().size());
assertEquals( original.require(id1.instance()).deploymentJobs().jobStatus().get(JobType.systemTest),
serialized.require(id1.instance()).deploymentJobs().jobStatus().get(JobType.systemTest));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializerTest.java
deleted file mode 100644
index 3bb5e3d0c2b..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/InstanceSerializerTest.java
+++ /dev/null
@@ -1,223 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.persistence;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.application.api.ValidationOverrides;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.config.SlimeUtils;
-import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
-import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
-import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
-import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
-import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
-import com.yahoo.vespa.hosted.controller.rotation.RotationId;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
-import org.junit.Test;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.OptionalDouble;
-import java.util.OptionalInt;
-import java.util.OptionalLong;
-import java.util.Set;
-
-import static com.yahoo.config.provision.SystemName.main;
-import static java.util.Optional.empty;
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author bratseth
- */
-public class InstanceSerializerTest {
-
- private static final InstanceSerializer INSTANCE_SERIALIZER = new InstanceSerializer();
- private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/");
- private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1");
- private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3");
-
- @Test
- public void testSerialization() {
- DeploymentSpec deploymentSpec = DeploymentSpec.fromXml("<deployment version='1.0'>" +
- " <staging/>" +
- "</deployment>");
- ValidationOverrides validationOverrides = ValidationOverrides.fromXml("<validation-overrides version='1.0'>" +
- " <allow until='2017-06-15'>deployment-removal</allow>" +
- "</validation-overrides>");
-
- List<Deployment> deployments = new ArrayList<>();
- ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31);
- ApplicationVersion applicationVersion2 = ApplicationVersion
- .from(new SourceRevision("repo1", "branch1", "commit1"), 32, "a@b",
- Version.fromString("6.3.1"), Instant.ofEpochMilli(496));
- Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z");
- deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils
- deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5),
- createClusterUtils(3, 0.2), createClusterInfo(3, 4),
- new DeploymentMetrics(2, 3, 4, 5, 6,
- Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)),
- Map.of(DeploymentMetrics.Warning.all, 3)),
- DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt),
- OptionalDouble.of(200), OptionalDouble.of(10))));
-
- OptionalLong projectId = OptionalLong.of(123L);
- List<JobStatus> statusList = new ArrayList<>();
-
- statusList.add(JobStatus.initial(JobType.systemTest)
- .withTriggering(Version.fromString("5.6.7"), ApplicationVersion.unknown, empty(), "Test", Instant.ofEpochMilli(7))
- .withCompletion(30, empty(), Instant.ofEpochMilli(8))
- .withPause(OptionalLong.of(1L << 32)));
- statusList.add(JobStatus.initial(JobType.stagingTest)
- .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, empty(), "Test 2", Instant.ofEpochMilli(5))
- .withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6)));
- statusList.add(JobStatus.initial(JobType.from(main, zone1).get())
- .withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, deployments.stream().findFirst(), "Test 3", Instant.ofEpochMilli(6))
- .withCompletion(11, empty(), Instant.ofEpochMilli(7)));
-
- DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty(), true);
-
- var rotationStatus = RotationStatus.from(Map.of(new RotationId("my-rotation"),
- Map.of(ZoneId.from("prod", "us-west-1"), RotationState.in,
- ZoneId.from("prod", "us-east-3"), RotationState.out)));
-
- Instance original = new Instance(ApplicationId.from("t1", "a1", "i1"),
- Instant.now().truncatedTo(ChronoUnit.MILLIS),
- deploymentSpec,
- validationOverrides,
- deployments, deploymentJobs,
- Change.of(Version.fromString("6.7")).withPin(),
- Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)),
- Optional.of(IssueId.from("1234")),
- Optional.of(User.from("by-username")),
- OptionalInt.of(7),
- new ApplicationMetrics(0.5, 0.9),
- Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----"),
- List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of())),
- rotationStatus);
-
- Instance serialized = INSTANCE_SERIALIZER.fromSlime(INSTANCE_SERIALIZER.toSlime(original));
-
- assertEquals(original.id(), serialized.id());
- assertEquals(original.createdAt(), serialized.createdAt());
-
- assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
- assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
-
- assertEquals(2, serialized.deployments().size());
- assertEquals(original.deployments().get(zone1).applicationVersion(), serialized.deployments().get(zone1).applicationVersion());
- assertEquals(original.deployments().get(zone2).applicationVersion(), serialized.deployments().get(zone2).applicationVersion());
- assertEquals(original.deployments().get(zone1).version(), serialized.deployments().get(zone1).version());
- assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version());
- assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at());
- assertEquals(original.deployments().get(zone2).at(), serialized.deployments().get(zone2).at());
- assertEquals(original.deployments().get(zone2).activity().lastQueried().get(), serialized.deployments().get(zone2).activity().lastQueried().get());
- assertEquals(original.deployments().get(zone2).activity().lastWritten().get(), serialized.deployments().get(zone2).activity().lastWritten().get());
-
- assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId());
- assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size());
- assertEquals( original.deploymentJobs().jobStatus().get(JobType.systemTest),
- serialized.deploymentJobs().jobStatus().get(JobType.systemTest));
- assertEquals( original.deploymentJobs().jobStatus().get(JobType.stagingTest),
- serialized.deploymentJobs().jobStatus().get(JobType.stagingTest));
-
- assertEquals(original.outstandingChange(), serialized.outstandingChange());
-
- assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId());
- assertEquals(original.owner(), serialized.owner());
- assertEquals(original.majorVersion(), serialized.majorVersion());
- assertEquals(original.change(), serialized.change());
- assertEquals(original.pemDeployKey(), serialized.pemDeployKey());
-
- assertEquals(original.rotations(), serialized.rotations());
- assertEquals(original.rotationStatus(), serialized.rotationStatus());
-
- // Test cluster utilization
- assertEquals(0, serialized.deployments().get(zone1).clusterUtils().size());
- assertEquals(3, serialized.deployments().get(zone2).clusterUtils().size());
- assertEquals(0.4, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id2")).getCpu(), 0.01);
- assertEquals(0.2, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getCpu(), 0.01);
- assertEquals(0.2, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getMemory(), 0.01);
-
- // Test cluster info
- assertEquals(3, serialized.deployments().get(zone2).clusterInfo().size());
- assertEquals(10, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost());
- assertEquals(ClusterSpec.Type.content, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType());
- assertEquals("flavor2", serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor());
- assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size());
- assertEquals(2, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE);
- assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE);
- assertEquals(50, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE);
-
- // Test metrics
- assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
- assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);
- assertEquals(original.deployments().get(zone2).metrics().queriesPerSecond(), serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
- assertEquals(original.deployments().get(zone2).metrics().writesPerSecond(), serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
- assertEquals(original.deployments().get(zone2).metrics().documentCount(), serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
- assertEquals(original.deployments().get(zone2).metrics().queryLatencyMillis(), serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
- assertEquals(original.deployments().get(zone2).metrics().writeLatencyMillis(), serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
- assertEquals(original.deployments().get(zone2).metrics().instant(), serialized.deployments().get(zone2).metrics().instant());
- assertEquals(original.deployments().get(zone2).metrics().warnings(), serialized.deployments().get(zone2).metrics().warnings());
- }
-
- private Map<ClusterSpec.Id, ClusterInfo> createClusterInfo(int clusters, int hosts) {
- Map<ClusterSpec.Id, ClusterInfo> result = new HashMap<>();
-
- for (int cluster = 0; cluster < clusters; cluster++) {
- List<String> hostnames = new ArrayList<>();
- for (int host = 0; host < hosts; host++) {
- hostnames.add("hostname" + cluster*host + host);
- }
-
- result.put(ClusterSpec.Id.from("id" + cluster), new ClusterInfo("flavor" + cluster, 10,
- 2, 4, 50, ClusterSpec.Type.content, hostnames));
- }
- return result;
- }
-
- private Map<ClusterSpec.Id, ClusterUtilization> createClusterUtils(int clusters, double inc) {
- Map<ClusterSpec.Id, ClusterUtilization> result = new HashMap<>();
-
- ClusterUtilization util = new ClusterUtilization(0,0,0,0);
- for (int cluster = 0; cluster < clusters; cluster++) {
- double agg = cluster*inc;
- result.put(ClusterSpec.Id.from("id" + cluster), new ClusterUtilization(
- util.getMemory()+ agg,
- util.getCpu()+ agg,
- util.getDisk() + agg,
- util.getDiskBusy() + agg));
- }
- return result;
- }
-
- @Test
- public void testCompleteApplicationDeserialization() throws Exception {
- byte[] applicationJson = Files.readAllBytes(testData.resolve("complete-instance.json"));
- INSTANCE_SERIALIZER.fromSlime(SlimeUtils.jsonToSlime(applicationJson));
- // ok if no error
- }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldCuratorDb.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldCuratorDb.java
deleted file mode 100644
index f7fdd91d74e..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldCuratorDb.java
+++ /dev/null
@@ -1,687 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.persistence;
-
-import com.google.common.util.concurrent.UncheckedTimeoutException;
-import com.google.inject.Inject;
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.path.Path;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
-import com.yahoo.vespa.curator.Curator;
-import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
-import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
-import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
-import com.yahoo.vespa.hosted.controller.deployment.Run;
-import com.yahoo.vespa.hosted.controller.deployment.Step;
-import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue;
-import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.versions.ControllerVersion;
-import com.yahoo.vespa.hosted.controller.versions.OsVersion;
-import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
-import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
-import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.nio.ByteBuffer;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-import java.util.stream.LongStream;
-import java.util.stream.Stream;
-
-import static java.util.Comparator.comparing;
-import static java.util.stream.Collectors.collectingAndThen;
-
-/**
- * Copy the current to replace this class before doing a data migration.
- * Use old serializers too, if needed.
- *
- * @author jonmv
- */
-public class OldCuratorDb {
-
- private static final Logger log = Logger.getLogger(CuratorDb.class.getName());
- private static final Duration deployLockTimeout = Duration.ofMinutes(30);
- private static final Duration defaultLockTimeout = Duration.ofMinutes(5);
- private static final Duration defaultTryLockTimeout = Duration.ofSeconds(1);
-
- private static final Path root = Path.fromString("/controller/v1");
- private static final Path lockRoot = root.append("locks");
- private static final Path tenantRoot = root.append("tenants");
- private static final Path applicationRoot = root.append("applications");
- private static final Path instanceRoot = root.append("instances");
- private static final Path jobRoot = root.append("jobs");
- private static final Path controllerRoot = root.append("controllers");
- private static final Path routingPoliciesRoot = root.append("routingPolicies");
- private static final Path applicationCertificateRoot = root.append("applicationCertificates");
-
- private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
- private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer();
- private final ControllerVersionSerializer controllerVersionSerializer = new ControllerVersionSerializer();
- private final ConfidenceOverrideSerializer confidenceOverrideSerializer = new ConfidenceOverrideSerializer();
- private final TenantSerializer tenantSerializer = new TenantSerializer();
- private final ApplicationSerializer applicationSerializer = new ApplicationSerializer();
- private final InstanceSerializer instanceSerializer = new InstanceSerializer();
- private final RunSerializer runSerializer = new RunSerializer();
- private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer();
- private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer);
- private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer();
- private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer();
- private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer();
-
- private final Curator curator;
- private final Duration tryLockTimeout;
-
- /**
- * All keys, to allow reentrancy.
- * This will grow forever, but this should be too slow to be a problem.
- */
- private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>();
-
- @Inject
- public OldCuratorDb(Curator curator) {
- this(curator, defaultTryLockTimeout);
- }
-
- OldCuratorDb(Curator curator, Duration tryLockTimeout) {
- this.curator = curator;
- this.tryLockTimeout = tryLockTimeout;
- }
-
- /** Returns all hosts configured to be part of this ZooKeeper cluster */
- public List<HostName> cluster() {
- return Arrays.stream(curator.zooKeeperEnsembleConnectionSpec().split(","))
- .filter(hostAndPort -> !hostAndPort.isEmpty())
- .map(hostAndPort -> hostAndPort.split(":")[0])
- .map(HostName::from)
- .collect(Collectors.toList());
- }
-
- // -------------- Locks ---------------------------------------------------
-
- /** Creates a reentrant lock */
- private Lock lock(Path path, Duration timeout) {
- curator.create(path);
- Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), curator));
- lock.acquire(timeout);
- return lock;
- }
-
- public Lock lock(TenantName name) {
- return lock(lockPath(name), defaultLockTimeout.multipliedBy(2));
- }
-
- public Lock lock(TenantAndApplicationId id) {
- return lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
- }
-
- public Lock lock(ApplicationId id) {
- return lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
- }
-
- public Lock lockForDeployment(ApplicationId id, ZoneId zone) {
- return lock(lockPath(id, zone), deployLockTimeout);
- }
-
- public Lock lock(ApplicationId id, JobType type) {
- return lock(lockPath(id, type), defaultLockTimeout);
- }
-
- public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException {
- return tryLock(lockPath(id, type, step));
- }
-
- public Lock lockRotations() {
- return lock(lockRoot.append("rotations"), defaultLockTimeout);
- }
-
- public Lock lockConfidenceOverrides() {
- return lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout);
- }
-
- public Lock lockInactiveJobs() {
- return lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout);
- }
-
- public Lock lockMaintenanceJob(String jobName) throws TimeoutException {
- return tryLock(lockRoot.append("maintenanceJobLocks").append(jobName));
- }
-
- @SuppressWarnings("unused") // Called by internal code
- public Lock lockProvisionState(String provisionStateId) {
- return lock(lockPath(provisionStateId), Duration.ofSeconds(1));
- }
-
- public Lock lockOsVersions() {
- return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout);
- }
-
- public Lock lockOsVersionStatus() {
- return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout);
- }
-
- public Lock lockRoutingPolicies() {
- return lock(lockRoot.append("routingPolicies"), defaultLockTimeout);
- }
-
- public Lock lockAuditLog() {
- return lock(lockRoot.append("auditLog"), defaultLockTimeout);
- }
-
- public Lock lockNameServiceQueue() {
- return lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout);
- }
-
- // -------------- Helpers ------------------------------------------
-
- /** Try locking with a low timeout, meaning it is OK to fail lock acquisition.
- *
- * Useful for maintenance jobs, where there is no point in running the jobs back to back.
- */
- private Lock tryLock(Path path) throws TimeoutException {
- try {
- return lock(path, tryLockTimeout);
- }
- catch (UncheckedTimeoutException e) {
- throw new TimeoutException(e.getMessage());
- }
- }
-
- private <T> Optional<T> read(Path path, Function<byte[], T> mapper) {
- return curator.getData(path).filter(data -> data.length > 0).map(mapper);
- }
-
- private Optional<Slime> readSlime(Path path) {
- return read(path, SlimeUtils::jsonToSlime);
- }
-
- private static byte[] asJson(Slime slime) {
- try {
- return SlimeUtils.toJsonBytes(slime);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- // -------------- Deployment orchestration --------------------------------
-
- public Set<String> readInactiveJobs() {
- try {
- return readSlime(inactiveJobsPath()).map(stringSetSerializer::fromSlime).orElseGet(HashSet::new);
- }
- catch (RuntimeException e) {
- log.log(Level.WARNING, "Error reading inactive jobs, deleting inactive state");
- writeInactiveJobs(Collections.emptySet());
- return new HashSet<>();
- }
- }
-
- public void writeInactiveJobs(Set<String> inactiveJobs) {
- curator.set(inactiveJobsPath(), stringSetSerializer.toJson(inactiveJobs));
- }
-
- public double readUpgradesPerMinute() {
- return read(upgradesPerMinutePath(), ByteBuffer::wrap).map(ByteBuffer::getDouble).orElse(0.125);
- }
-
- public void writeUpgradesPerMinute(double n) {
- curator.set(upgradesPerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array());
- }
-
- public Optional<Integer> readTargetMajorVersion() {
- return read(targetMajorVersionPath(), ByteBuffer::wrap).map(ByteBuffer::getInt);
- }
-
- public void writeTargetMajorVersion(Optional<Integer> targetMajorVersion) {
- if (targetMajorVersion.isPresent())
- curator.set(targetMajorVersionPath(), ByteBuffer.allocate(Integer.BYTES).putInt(targetMajorVersion.get()).array());
- else
- curator.delete(targetMajorVersionPath());
- }
-
- public void writeVersionStatus(VersionStatus status) {
- curator.set(versionStatusPath(), asJson(versionStatusSerializer.toSlime(status)));
- }
-
- public VersionStatus readVersionStatus() {
- return readSlime(versionStatusPath()).map(versionStatusSerializer::fromSlime).orElseGet(VersionStatus::empty);
- }
-
- public void writeConfidenceOverrides(Map<Version, VespaVersion.Confidence> overrides) {
- curator.set(confidenceOverridesPath(), asJson(confidenceOverrideSerializer.toSlime(overrides)));
- }
-
- public Map<Version, VespaVersion.Confidence> readConfidenceOverrides() {
- return readSlime(confidenceOverridesPath()).map(confidenceOverrideSerializer::fromSlime)
- .orElseGet(Collections::emptyMap);
- }
-
- public void writeControllerVersion(HostName hostname, ControllerVersion version) {
- curator.set(controllerPath(hostname.value()), asJson(controllerVersionSerializer.toSlime(version)));
- }
-
- public ControllerVersion readControllerVersion(HostName hostname) {
- return readSlime(controllerPath(hostname.value()))
- .map(controllerVersionSerializer::fromSlime)
- .orElse(ControllerVersion.CURRENT);
- }
-
- // Infrastructure upgrades
-
- public void writeOsVersions(Set<OsVersion> versions) {
- curator.set(osTargetVersionPath(), asJson(osVersionSerializer.toSlime(versions)));
- }
-
- public Set<OsVersion> readOsVersions() {
- return readSlime(osTargetVersionPath()).map(osVersionSerializer::fromSlime).orElseGet(Collections::emptySet);
- }
-
- public void writeOsVersionStatus(OsVersionStatus status) {
- curator.set(osVersionStatusPath(), asJson(osVersionStatusSerializer.toSlime(status)));
- }
-
- public OsVersionStatus readOsVersionStatus() {
- return readSlime(osVersionStatusPath()).map(osVersionStatusSerializer::fromSlime).orElse(OsVersionStatus.empty);
- }
-
- // -------------- Tenant --------------------------------------------------
-
- public void writeTenant(Tenant tenant) {
- curator.set(tenantPath(tenant.name()), asJson(tenantSerializer.toSlime(tenant)));
- }
-
- public Optional<Tenant> readTenant(TenantName name) {
- return readSlime(tenantPath(name)).map(tenantSerializer::tenantFrom);
- }
-
- public List<Tenant> readTenants() {
- return readTenantNames().stream()
- .map(this::readTenant)
- .flatMap(Optional::stream)
- .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
- }
-
- public List<TenantName> readTenantNames() {
- return curator.getChildren(tenantRoot).stream()
- .map(TenantName::from)
- .collect(Collectors.toList());
- }
-
- public void removeTenant(TenantName name) {
- curator.delete(tenantPath(name));
- }
-
- // -------------- Applications ---------------------------------------------
-
- public void writeApplication(Application application) {
- curator.set(applicationPath(application.id()), asJson(applicationSerializer.toSlime(application)));
- for (InstanceName name : application.instances().keySet()) {
- curator.set(oldApplicationPath(application.id().instance(name)),
- asJson(instanceSerializer.toSlime(application.legacy(name))));
- }
- }
-
- public Optional<Application> readApplication(TenantAndApplicationId application) {
- List<Instance> instances = readInstances(id -> TenantAndApplicationId.from(id).equals(application));
- return Application.aggregate(instances);
- }
-
- public List<Application> readApplications() {
- return readApplications(ignored -> true);
- }
-
- public List<Application> readApplications(TenantName name) {
- return readApplications(application -> application.tenant().equals(name));
- }
-
- private Stream<TenantAndApplicationId> readTenantAndApplicationIds() {
- return readInstanceIds().map(TenantAndApplicationId::from).distinct();
- }
-
- private List<Application> readApplications(Predicate<TenantAndApplicationId> applicationFilter) {
- return readTenantAndApplicationIds().filter(applicationFilter)
- .sorted()
- .map(this::readApplication)
- .flatMap(Optional::stream)
- .collect(Collectors.toUnmodifiableList());
- }
-
- private Optional<Instance> readInstance(ApplicationId application) {
- return readSlime(oldApplicationPath(application)).map(instanceSerializer::fromSlime);
- }
-
- private Stream<ApplicationId> readInstanceIds() {
- return curator.getChildren(applicationRoot).stream()
- .filter(id -> id.split(":").length == 3)
- .map(ApplicationId::fromSerializedForm);
- }
-
- private List<Instance> readInstances(Predicate<ApplicationId> applicationFilter) {
- return readInstanceIds().filter(applicationFilter)
- .sorted()
- .map(this::readInstance)
- .flatMap(Optional::stream)
- .collect(Collectors.toUnmodifiableList());
- }
-
- public void removeApplication(ApplicationId id) {
- // WARNING: This is part of a multi-step data move operation, so don't touch!!!
- curator.delete(oldApplicationPath(id));
- if (readApplication(TenantAndApplicationId.from(id)).isEmpty())
- curator.delete(applicationPath(TenantAndApplicationId.from(id)));
- }
-
- public void clearInstanceRoot() {
- curator.delete(instanceRoot);
- }
-
- /**
- * Migration plan:
- *
- * Add filter for reading only Instance from old application path RELEASED
- * Write Instance to instance and old application path RELEASED
- *
- * Lock on application level for instance mutations MERGED
- *
- * Write Instance to instance and application and old application paths DONE TO CHANGE DONE
- * Read Instance from instance path DONE TO REMOVE DONE
- * Duplicate Application from Instance, with helper classes DONE
- * Write Application to instance and application and old application paths DONE TO CHANGE DONE
- * Read Application from instance path DONE TO REMOVE DONE
- * Use Application where applicable DONE !!!
- * Lock instances and application on same level: tenant + application DONE TO CHANGE DONE
- * When reading an application, read all instances, and aggregate them DONE
- * Write application with instances to application path DONE
- * Write all instances of an application to old application path DONE
- * Remove everything under instance root DONE
- *
- * Read Application with instances from application path (with filter)
- * Stop locking applications on instance level
- *
- * Stop writing Instance to old application path
- * Remove unused parts of Instance (Used only for legacy serialization)
- */
-
- // -------------- Job Runs ------------------------------------------------
-
- public void writeLastRun(Run run) {
- curator.set(lastRunPath(run.id().application(), run.id().type()), asJson(runSerializer.toSlime(run)));
- }
-
- public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) {
- curator.set(runsPath(id, type), asJson(runSerializer.toSlime(runs)));
- }
-
- public Optional<Run> readLastRun(ApplicationId id, JobType type) {
- return readSlime(lastRunPath(id, type)).map(runSerializer::runFromSlime);
- }
-
- public SortedMap<RunId, Run> readHistoricRuns(ApplicationId id, JobType type) {
- return readSlime(runsPath(id, type)).map(runSerializer::runsFromSlime).orElse(new TreeMap<>(comparing(RunId::number)));
- }
-
- public void deleteRunData(ApplicationId id, JobType type) {
- curator.delete(runsPath(id, type));
- curator.delete(lastRunPath(id, type));
- }
-
- public void deleteRunData(ApplicationId id) {
- curator.delete(jobRoot.append(id.serializedForm()));
- }
-
- public List<ApplicationId> applicationsWithJobs() {
- return curator.getChildren(jobRoot).stream()
- .map(ApplicationId::fromSerializedForm)
- .collect(Collectors.toList());
- }
-
-
- public Optional<byte[]> readLog(ApplicationId id, JobType type, long chunkId) {
- return curator.getData(logPath(id, type, chunkId));
- }
-
- public void writeLog(ApplicationId id, JobType type, long chunkId, byte[] log) {
- curator.set(logPath(id, type, chunkId), log);
- }
-
- public void deleteLog(ApplicationId id, JobType type) {
- curator.delete(runsPath(id, type).append("logs"));
- }
-
- public Optional<Long> readLastLogEntryId(ApplicationId id, JobType type) {
- return curator.getData(lastLogPath(id, type))
- .map(String::new).map(Long::parseLong);
- }
-
- public void writeLastLogEntryId(ApplicationId id, JobType type, long lastId) {
- curator.set(lastLogPath(id, type), Long.toString(lastId).getBytes());
- }
-
- public LongStream getLogChunkIds(ApplicationId id, JobType type) {
- return curator.getChildren(runsPath(id, type).append("logs")).stream()
- .mapToLong(Long::parseLong)
- .sorted();
- }
-
- // -------------- Audit log -----------------------------------------------
-
- public AuditLog readAuditLog() {
- return readSlime(auditLogPath()).map(auditLogSerializer::fromSlime)
- .orElse(AuditLog.empty);
- }
-
- public void writeAuditLog(AuditLog log) {
- curator.set(auditLogPath(), asJson(auditLogSerializer.toSlime(log)));
- }
-
-
- // -------------- Name service log ----------------------------------------
-
- public NameServiceQueue readNameServiceQueue() {
- return readSlime(nameServiceQueuePath()).map(nameServiceQueueSerializer::fromSlime)
- .orElse(NameServiceQueue.EMPTY);
- }
-
- public void writeNameServiceQueue(NameServiceQueue queue) {
- curator.set(nameServiceQueuePath(), asJson(nameServiceQueueSerializer.toSlime(queue)));
- }
-
- // -------------- Provisioning (called by internal code) ------------------
-
- @SuppressWarnings("unused")
- public Optional<byte[]> readProvisionState(String provisionId) {
- return curator.getData(provisionStatePath(provisionId));
- }
-
- @SuppressWarnings("unused")
- public void writeProvisionState(String provisionId, byte[] data) {
- curator.set(provisionStatePath(provisionId), data);
- }
-
- @SuppressWarnings("unused")
- public List<String> readProvisionStateIds() {
- return curator.getChildren(provisionStatePath());
- }
-
- // -------------- Routing policies ----------------------------------------
-
- public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) {
- curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies)));
- }
-
- public Map<ApplicationId, Set<RoutingPolicy>> readRoutingPolicies() {
- return curator.getChildren(routingPoliciesRoot).stream()
- .map(ApplicationId::fromSerializedForm)
- .collect(Collectors.toUnmodifiableMap(Function.identity(), this::readRoutingPolicies));
- }
-
- public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) {
- return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime))
- .orElseGet(Collections::emptySet);
- }
-
- // -------------- Application web certificates ----------------------------
-
- public void writeApplicationCertificate(ApplicationId applicationId, ApplicationCertificate applicationCertificate) {
- curator.set(applicationCertificatePath(applicationId), applicationCertificate.secretsKeyNamePrefix().getBytes());
- }
-
- public Optional<ApplicationCertificate> readApplicationCertificate(ApplicationId applicationId) {
- return curator.getData(applicationCertificatePath(applicationId)).map(String::new).map(ApplicationCertificate::new);
- }
-
- // -------------- Paths ---------------------------------------------------
-
- private Path lockPath(TenantName tenant) {
- return lockRoot
- .append(tenant.value());
- }
-
- private Path lockPath(TenantAndApplicationId application) {
- return lockPath(application.tenant())
- .append(application.application().value());
- }
-
- private Path lockPath(ApplicationId instance) {
- return lockPath(TenantAndApplicationId.from(instance))
- .append(instance.instance().value());
- }
-
- private Path lockPath(ApplicationId application, ZoneId zone) {
- return lockPath(application)
- .append(zone.environment().value())
- .append(zone.region().value());
- }
-
- private Path lockPath(ApplicationId application, JobType type) {
- return lockPath(application)
- .append(type.jobName());
- }
-
- private Path lockPath(ApplicationId application, JobType type, Step step) {
- return lockPath(application, type)
- .append(step.name());
- }
-
- private Path lockPath(String provisionId) {
- return lockRoot
- .append(provisionStatePath())
- .append(provisionId);
- }
-
- private static Path inactiveJobsPath() {
- return root.append("inactiveJobs");
- }
-
- private static Path upgradesPerMinutePath() {
- return root.append("upgrader").append("upgradesPerMinute");
- }
-
- private static Path targetMajorVersionPath() {
- return root.append("upgrader").append("targetMajorVersion");
- }
-
- private static Path confidenceOverridesPath() {
- return root.append("upgrader").append("confidenceOverrides");
- }
-
- private static Path osTargetVersionPath() {
- return root.append("osUpgrader").append("targetVersion");
- }
-
- private static Path osVersionStatusPath() {
- return root.append("osVersionStatus");
- }
-
- private static Path versionStatusPath() {
- return root.append("versionStatus");
- }
-
- private static Path routingPolicyPath(ApplicationId application) {
- return routingPoliciesRoot.append(application.serializedForm());
- }
-
- private static Path nameServiceQueuePath() {
- return root.append("nameServiceQueue");
- }
-
- private static Path auditLogPath() {
- return root.append("auditLog");
- }
-
- private static Path provisionStatePath() {
- return root.append("provisioning").append("states");
- }
-
- private static Path provisionStatePath(String provisionId) {
- return provisionStatePath().append(provisionId);
- }
-
- private static Path tenantPath(TenantName name) {
- return tenantRoot.append(name.value());
- }
-
- private static Path applicationPath(TenantAndApplicationId id) {
- return applicationRoot.append(id.serialized());
- }
-
- private static Path oldApplicationPath(ApplicationId application) {
- return applicationRoot.append(application.serializedForm());
- }
-
- private static Path instancePath(ApplicationId id) {
- return instanceRoot.append(id.serializedForm());
- }
-
- private static Path runsPath(ApplicationId id, JobType type) {
- return jobRoot.append(id.serializedForm()).append(type.jobName());
- }
-
- private static Path lastRunPath(ApplicationId id, JobType type) {
- return runsPath(id, type).append("last");
- }
-
- private static Path logPath(ApplicationId id, JobType type, long first) {
- return runsPath(id, type).append("logs").append(Long.toString(first));
- }
-
- private static Path lastLogPath(ApplicationId id, JobType type) {
- return runsPath(id, type).append("logs");
- }
-
- private static Path controllerPath(String hostname) {
- return controllerRoot.append(hostname);
- }
-
- private static Path applicationCertificatePath(ApplicationId id) {
- return applicationCertificateRoot.append(id.serializedForm());
- }
-
-}
-
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldMockCuratorDb.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldMockCuratorDb.java
deleted file mode 100644
index 341fe05f67d..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OldMockCuratorDb.java
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.persistence;
-
-import com.google.inject.Inject;
-import com.yahoo.vespa.curator.Curator;
-import com.yahoo.vespa.curator.mock.MockCurator;
-
-import java.time.Duration;
-
-/**
- * A curator db backed by a mock curator.
- *
- * @author bratseth
- */
-@SuppressWarnings("unused") // injected
-public class OldMockCuratorDb extends OldCuratorDb {
-
- public OldMockCuratorDb(MockCurator curator) {
- super(curator, Duration.ofMillis(100));
- }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-instance.json
deleted file mode 100644
index 28f505e88ec..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-instance.json
+++ /dev/null
@@ -1,534 +0,0 @@
-{
- "id": "tenant1:app1:default",
- "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <!--<staging />-->\n <prod global-service-id=\"foo\">\n <region active=\"true\">us-east-3</region>\n <region active=\"true\">us-west-1</region>\n </prod>\n</deployment>\n",
- "validationOverrides": "<validation-overrides>\n <allow until=\"2016-04-28\" comment=\"Renaming content cluster\">content-cluster-removal</allow>\n <allow until=\"2016-08-22\" comment=\"Migrating us-east-3 to C-2E\">cluster-size-reduction</allow>\n <allow until=\"2017-06-30\" comment=\"Test Vespa upgrade tests\">force-automatic-tenant-upgrade-test</allow>\n</validation-overrides>\n",
- "deployments": [
- {
- "zone": {
- "environment": "prod",
- "region": "us-west-1"
- },
- "version": "6.173.62",
- "deployTime": 1510837817704,
- "applicationPackageRevision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "clusterInfo": {
- "cluster1": {
- "flavor": "d-3-16-100",
- "cost": 9,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "container",
- "hostnames": [
- "node1",
- "node2"
- ]
- },
- "cluster2": {
- "flavor": "d-12-64-400",
- "cost": 38,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "content",
- "hostnames": [
- "node3",
- "node4",
- "node5"
- ]
- },
- "cluster3": {
- "flavor": "d-12-64-400",
- "cost": 38,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "content",
- "hostnames": [
- "node6",
- "node7",
- "node8",
- "node9"
- ]
- }
- },
- "clusterUtils": {
- "cluster1": {
- "cpu": 0.1720353499228221,
- "mem": 0.4986146831512451,
- "disk": 0.0617671330041831,
- "diskbusy": 0
- },
- "cluster2": {
- "cpu": 0.07505730001866318,
- "mem": 0.7936344432830811,
- "disk": 0.2260549694485994,
- "diskbusy": 0
- },
- "cluster3": {
- "cpu": 0.01712671480989384,
- "mem": 0.0225852754983035,
- "disk": 0.006084436856721915,
- "diskbusy": 0
- }
- },
- "metrics": {
- "queriesPerSecond": 1.25,
- "writesPerSecond": 43.83199977874756,
- "documentCount": 525880277.9999999,
- "queryLatencyMillis": 5.607503938674927,
- "writeLatencyMillis": 20.57866265104621
- }
- },
- {
- "zone": {
- "environment": "test",
- "region": "us-east-1"
- },
- "version": "6.173.62",
- "deployTime": 1511256872316,
- "applicationPackageRevision": {
- "applicationPackageHash": "ec548fa61cbfab7a270a51d46b1263ec1be5d9a8",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "234f3e4e77049d0b9538c9e1b356d17eb1dedb6a"
- }
- },
- "clusterInfo": {},
- "clusterUtils": {},
- "metrics": {
- "queriesPerSecond": 0,
- "writesPerSecond": 0,
- "documentCount": 0,
- "queryLatencyMillis": 0,
- "writeLatencyMillis": 0
- }
- },
- {
- "zone": {
- "environment": "dev",
- "region": "us-east-1"
- },
- "version": "6.173.62",
- "deployTime": 1510597489464,
- "applicationPackageRevision": {
- "applicationPackageHash": "59b883f263c2a3c23dfab249730097d7e0e1ed32"
- },
- "clusterInfo": {
- "cluster1": {
- "flavor": "d-2-8-50",
- "cost": 5,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "container",
- "hostnames": [
- "node1"
- ]
- },
- "cluster2": {
- "flavor": "d-2-8-50",
- "cost": 5,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "content",
- "hostnames": [
- "node2"
- ]
- },
- "cluster3": {
- "flavor": "d-2-8-50",
- "cost": 5,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "content",
- "hostnames": [
- "node3"
- ]
- }
- },
- "clusterUtils": {
- "cluster1": {
- "cpu": 0.191833330678661,
- "mem": 0.4625738318415235,
- "disk": 0.05582004563850269,
- "diskbusy": 0
- },
- "cluster2": {
- "cpu": 0.2227037978608054,
- "mem": 0.2051752598416401,
- "disk": 0.05471533698695047,
- "diskbusy": 0
- },
- "cluster3": {
- "cpu": 0.1869410834020498,
- "mem": 0.1691722576000564,
- "disk": 0.04977374774258153,
- "diskbusy": 0
- }
- },
- "metrics": {
- "queriesPerSecond": 0,
- "writesPerSecond": 0,
- "documentCount": 30916,
- "queryLatencyMillis": 0,
- "writeLatencyMillis": 0
- }
- },
- {
- "zone": {
- "environment": "prod",
- "region": "us-east-3"
- },
- "version": "6.173.62",
- "deployTime": 1510817190016,
- "applicationPackageRevision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "clusterInfo": {
- "cluster1": {
- "flavor": "d-3-16-100",
- "cost": 9,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "container",
- "hostnames": [
- "node1",
- "node2"
- ]
- },
- "cluster2": {
- "flavor": "d-12-64-400",
- "cost": 38,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "content",
- "hostnames": [
- "node1",
- "node2",
- "node3"
- ]
- },
- "cluster3": {
- "flavor": "d-12-64-400",
- "cost": 38,
- "flavorCpu": 0,
- "flavorMem": 0,
- "flavorDisk": 0,
- "clusterType": "content",
- "hostnames": [
- "node1",
- "node2",
- "node3",
- "node4"
- ]
- }
- },
- "clusterUtils": {
- "cluster1": {
- "cpu": 0.2295038983007097,
- "mem": 0.4627357390237263,
- "disk": 0.05559941525894966,
- "diskbusy": 0
- },
- "cluster2": {
- "cpu": 0.05340429087579549,
- "mem": 0.8107630891552372,
- "disk": 0.226444914138854,
- "diskbusy": 0
- },
- "cluster3": {
- "cpu": 0.02148227413975218,
- "mem": 0.02162174219104161,
- "disk": 0.006057760545243265,
- "diskbusy": 0
- }
- },
- "metrics": {
- "queriesPerSecond": 1.734000012278557,
- "writesPerSecond": 44.59999895095825,
- "documentCount": 525868193.9999999,
- "queryLatencyMillis": 5.65284947195106,
- "writeLatencyMillis": 17.34593812832452
- }
- }
- ],
- "deploymentJobs": {
- "projectId": 102889,
- "jobStatus": [
- {
- "jobType": "staging-test",
- "lastTriggered": {
- "id": -1,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "system-test completed",
- "at": 1510830134259
- },
- "lastCompleted": {
- "id": 1184,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "system-test completed",
- "at": 1510830684960
- },
- "lastSuccess": {
- "id": 1184,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "system-test completed",
- "at": 1510830684960
- }
- },
- {
- "jobType": "component",
- "lastCompleted": {
- "id": 849,
- "version": "6.174.156",
- "upgrade": false,
- "reason": "Application commit",
- "at": 1511217733555
- },
- "lastSuccess": {
- "id": 849,
- "version": "6.174.156",
- "upgrade": false,
- "reason": "Application commit",
- "at": 1511217733555
- }
- },
- {
- "jobType": "production-us-east-3",
- "lastTriggered": {
- "id": -1,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "staging-test completed",
- "at": 1510830685127
- },
- "lastCompleted": {
- "id": 923,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "staging-test completed",
- "at": 1510837650046
- },
- "lastSuccess": {
- "id": 923,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "staging-test completed",
- "at": 1510837650046
- }
- },
- {
- "jobType": "production-us-west-1",
- "lastTriggered": {
- "id": -1,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "production-us-east-3 completed",
- "at": 1510837650139
- },
- "lastCompleted": {
- "id": 646,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "production-us-east-3 completed",
- "at": 1510843559162
- },
- "lastSuccess": {
- "id": 646,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "production-us-east-3 completed",
- "at": 1510843559162
- }
- },
- {
- "jobType": "system-test",
- "jobError": "unknown",
- "lastTriggered": {
- "id": -1,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "ec548fa61cbfab7a270a51d46b1263ec1be5d9a8",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "234f3e4e77049d0b9538c9e1b356d17eb1dedb6a"
- }
- },
- "upgrade": false,
- "reason": "Available change in component",
- "at": 1511256608649
- },
- "lastCompleted": {
- "id": 1686,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "ec548fa61cbfab7a270a51d46b1263ec1be5d9a8",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "234f3e4e77049d0b9538c9e1b356d17eb1dedb6a"
- }
- },
- "upgrade": false,
- "reason": "Available change in component",
- "at": 1511256603353
- },
- "firstFailing": {
- "id": 1659,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "ec548fa61cbfab7a270a51d46b1263ec1be5d9a8",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "234f3e4e77049d0b9538c9e1b356d17eb1dedb6a"
- }
- },
- "upgrade": false,
- "reason": "component completed",
- "at": 1511219070725
- },
- "lastSuccess": {
- "id": 1658,
- "version": "6.173.62",
- "revision": {
- "applicationPackageHash": "9db423e1021d7b452d37ec6372bc757d9c1bda87",
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "49cd7bbb1ed9f4b922083cb042590b0885ffe22b"
- }
- },
- "upgrade": true,
- "reason": "Upgrading to 6.173.62",
- "at": 1511175754163
- }
- }
- ]
- },
- "deployingField": {
- "buildNumber": 42,
- "sourceRevision": {
- "repositoryField": "git@git.host:user/repo.git",
- "branchField": "origin/master",
- "commitField": "234f3e4e77049d0b9538c9e1b356d17eb1dedb6a"
- }
- },
- "outstandingChangeField": false,
- "queryQuality": 100,
- "writeQuality": 99.99894341115082,
- "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----",
- "assignedRotations": [
- {
- "rotationId": "rotation-foo",
- "clusterId": "qrs",
- "endpointId": "default"
- }
- ],
- "rotationStatus2": [
- {
- "rotationId": "rotation-foo",
- "status": [
- {
- "environment": "prod",
- "region": "us-east-3",
- "state": "in"
- }
- ]
- }
- ]
-}