// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.persistence;
import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.ValidationOverrides;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.CloudAccount;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.security.KeyUtils;
import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Instance;
import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.TokenId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
import com.yahoo.vespa.hosted.controller.api.integration.organization.AccountId;
import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory;
import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId;
import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState;
import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus;
import org.junit.jupiter.api.Test;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.PublicKey;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalDouble;
import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* @author bratseth
*/
public class ApplicationSerializerTest {
private static final ApplicationSerializer APPLICATION_SERIALIZER = new ApplicationSerializer();
private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/");
private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1");
private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3");
private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("""
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9
z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==
-----END PUBLIC KEY-----
""");
private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("""
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE
pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==
-----END PUBLIC KEY-----
""");
@Test
void testSerialization() throws Exception {
DeploymentSpec deploymentSpec = DeploymentSpec.fromXml("""
us-west-1
""");
ValidationOverrides validationOverrides = ValidationOverrides.fromXml("" +
" deployment-removal" +
"");
OptionalLong projectId = OptionalLong.of(123L);
ApplicationId id1 = ApplicationId.from("t1", "a1", "i1");
ApplicationId id3 = ApplicationId.from("t1", "a1", "i3");
List deployments = new ArrayList<>();
ApplicationVersion applicationVersion1 = new ApplicationVersion(RevisionId.forProduction(31),
Optional.of(new SourceRevision("git@github:org/repo.git", "branch1", "commit1")),
Optional.of("william@shakespeare"),
Optional.of(Version.fromString("1.2.3")),
Optional.of(123),
Optional.of(Instant.ofEpochMilli(666)),
Optional.empty(),
Optional.of("best commit"),
Optional.of("hash1"),
Optional.of(Instant.ofEpochMilli(777)),
true,
false,
Optional.of("~(˘▾˘)~"),
Optional.of(Instant.ofEpochMilli(496)),
3);
assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get());
RevisionId id = RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3));
SourceRevision source = new SourceRevision("repo1", "branch1", "commit1");
Version compileVersion = Version.fromString("6.3.1");
ApplicationVersion applicationVersion2 = new ApplicationVersion(id, Optional.of(source), Optional.of("a@b"), Optional.of(compileVersion), Optional.empty(), Optional.of(Instant.ofEpochMilli(496)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), Optional.empty(), 0);
Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z");
deployments.add(new Deployment(zone1, CloudAccount.empty, applicationVersion1.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(3),
DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty(), Map.of(TokenId.of("foo"), Instant.ofEpochMilli(333))));
deployments.add(new Deployment(zone2, CloudAccount.from("001122334455"), applicationVersion2.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(5),
new DeploymentMetrics(2, 3, 4, 5, 6,
Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)),
Map.of(DeploymentMetrics.Warning.all, 3)),
DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt),
OptionalDouble.of(200), OptionalDouble.of(10)),
QuotaUsage.create(OptionalDouble.of(23.5)),
OptionalDouble.of(12.3), Map.of()));
var rotationStatus = RotationStatus.from(Map.of(new RotationId("my-rotation"),
new RotationStatus.Targets(
Map.of(ZoneId.from("prod", "us-west-1"), RotationState.in,
ZoneId.from("prod", "us-east-3"), RotationState.out),
Instant.ofEpochMilli(42))));
RevisionHistory revisions = RevisionHistory.ofRevisions(List.of(applicationVersion1),
Map.of(new JobId(id1, DeploymentContext.productionUsEast3), List.of(applicationVersion2)));
List instances =
List.of(new Instance(id1,
deployments,
Map.of(DeploymentContext.systemTest, Instant.ofEpochMilli(333)),
List.of(rotation("foo", "default", "my-rotation", Set.of("us-west-1"))),
rotationStatus,
Change.of(new Version("6.1"))),
new Instance(id3,
List.of(),
Map.of(),
List.of(),
RotationStatus.EMPTY,
Change.of(Version.fromString("6.7")).withPlatformPin().withRevisionPin()));
Application original = new Application(TenantAndApplicationId.from(id1),
Instant.now().truncatedTo(ChronoUnit.MILLIS),
deploymentSpec,
validationOverrides,
Optional.of(IssueId.from("4321")),
Optional.of(IssueId.from("1234")),
Optional.of(User.from("by-username")),
Optional.of(new AccountId("foo8ar")),
OptionalInt.of(7),
new ApplicationMetrics(0.5, 0.9),
Set.of(publicKey, otherPublicKey),
projectId,
revisions,
instances
);
Application serialized = APPLICATION_SERIALIZER.fromSlime(SlimeUtils.toJsonBytes(APPLICATION_SERIALIZER.toSlime(original)));
assertEquals(original.id(), serialized.id());
assertEquals(original.createdAt(), serialized.createdAt());
assertEquals(applicationVersion1, serialized.revisions().last().get());
assertEquals(applicationVersion1, serialized.revisions().get(serialized.instances().get(id1.instance()).deployments().get(zone1).revision()));
assertEquals(original.revisions().last(), serialized.revisions().last());
assertEquals(original.revisions().last().get().compileVersion(), serialized.revisions().last().get().compileVersion());
assertEquals(original.revisions().last().get().allowedMajor(), serialized.revisions().last().get().allowedMajor());
assertEquals(original.revisions().last().get().authorEmail(), serialized.revisions().last().get().authorEmail());
assertEquals(original.revisions().last().get().buildTime(), serialized.revisions().last().get().buildTime());
assertEquals(original.revisions().last().get().sourceUrl(), serialized.revisions().last().get().sourceUrl());
assertEquals(original.revisions().last().get().commit(), serialized.revisions().last().get().commit());
assertEquals(original.revisions().last().get().bundleHash(), serialized.revisions().last().get().bundleHash());
assertEquals(original.revisions().last().get().obsoleteAt(), serialized.revisions().last().get().obsoleteAt());
assertEquals(original.revisions().last().get().hasPackage(), serialized.revisions().last().get().hasPackage());
assertEquals(original.revisions().last().get().shouldSkip(), serialized.revisions().last().get().shouldSkip());
assertEquals(original.revisions().last().get().description(), serialized.revisions().last().get().description());
assertEquals(original.revisions().last().get().submittedAt(), serialized.revisions().last().get().submittedAt());
assertEquals(original.revisions().last().get().risk(), serialized.revisions().last().get().risk());
assertEquals(original.revisions().withPackage(), serialized.revisions().withPackage());
assertEquals(original.revisions().production(), serialized.revisions().production());
assertEquals(original.revisions().development(), serialized.revisions().development());
assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
assertEquals(original.projectId(), serialized.projectId());
assertEquals(original.deploymentIssueId(), serialized.deploymentIssueId());
assertEquals(0, serialized.require(id3.instance()).deployments().size());
assertEquals(0, serialized.require(id3.instance()).rotations().size());
assertEquals(RotationStatus.EMPTY, serialized.require(id3.instance()).rotationStatus());
assertEquals(2, serialized.require(id1.instance()).deployments().size());
assertEquals(original.require(id1.instance()).deployments().get(zone1), serialized.require(id1.instance()).deployments().get(zone1));
assertEquals(original.require(id1.instance()).deployments().get(zone2), serialized.require(id1.instance()).deployments().get(zone2));
assertEquals(original.require(id1.instance()).jobPause(DeploymentContext.systemTest),
serialized.require(id1.instance()).jobPause(DeploymentContext.systemTest));
assertEquals(original.require(id1.instance()).jobPause(DeploymentContext.stagingTest),
serialized.require(id1.instance()).jobPause(DeploymentContext.stagingTest));
assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId());
assertEquals(original.userOwner(), serialized.userOwner());
assertEquals(original.issueOwner(), serialized.issueOwner());
assertEquals(original.majorVersion(), serialized.majorVersion());
assertEquals(original.deployKeys(), serialized.deployKeys());
assertEquals(original.require(id1.instance()).rotations(), serialized.require(id1.instance()).rotations());
assertEquals(original.require(id1.instance()).rotationStatus(), serialized.require(id1.instance()).rotationStatus());
assertEquals(original.require(id1.instance()).change(), serialized.require(id1.instance()).change());
assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change());
// Test metrics
assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().queriesPerSecond(), serialized.require(id1.instance()).deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().writesPerSecond(), serialized.require(id1.instance()).deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().documentCount(), serialized.require(id1.instance()).deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().queryLatencyMillis(), serialized.require(id1.instance()).deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().writeLatencyMillis(), serialized.require(id1.instance()).deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().instant(), serialized.require(id1.instance()).deployments().get(zone2).metrics().instant());
assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().warnings(), serialized.require(id1.instance()).deployments().get(zone2).metrics().warnings());
// Test quota
assertEquals(original.require(id1.instance()).deployments().get(zone2).quota().rate(), serialized.require(id1.instance()).deployments().get(zone2).quota().rate(), 0.001);
assertEquals(original.require(id1.instance()).deployments().get(zone1).dataPlaneTokens(), serialized.require(id1.instance()).deployments().get(zone1).dataPlaneTokens());
assertEquals(original.require(id1.instance()).deployments().get(zone2).dataPlaneTokens(), serialized.require(id1.instance()).deployments().get(zone2).dataPlaneTokens());
}
@Test
void testCompleteApplicationDeserialization() throws Exception {
byte[] applicationJson = Files.readAllBytes(testData.resolve("complete-application.json"));
APPLICATION_SERIALIZER.fromSlime(applicationJson);
// ok if no error
}
private static AssignedRotation rotation(String clusterId, String endpointId, String rotationId, Collection regions) {
return new AssignedRotation(
new ClusterSpec.Id(clusterId),
EndpointId.of(endpointId),
new RotationId(rotationId),
regions.stream().map(RegionName::from).collect(Collectors.toSet())
);
}
}