diff options
218 files changed, 3444 insertions, 1235 deletions
diff --git a/application/pom.xml b/application/pom.xml index cd27b53f557..96250acb5a9 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -54,6 +54,12 @@ <artifactId>container-search-gui</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> <!-- Because these are provided in jdisc_http_service and just preinstalled: --> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -86,6 +92,11 @@ <artifactId>icu4j</artifactId> </dependency> <dependency> + <groupId>io.airlift</groupId> + <artifactId>aircompressor</artifactId> + <scope>compile</scope> + </dependency> + <dependency> <groupId>com.optimaize.languagedetector</groupId> <artifactId>language-detector</artifactId> <exclusions> diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index e5f1d7737a1..0cc3f04a57b 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -134,6 +134,12 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java index 74f3cc276a5..f46badf2fd9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java @@ -10,7 +10,6 @@ import com.yahoo.vdslib.state.State; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.TreeMap; /** @@ -26,7 +25,7 @@ public class ClusterStateGenerator { static class Params { public ContentCluster cluster; - public Map<NodeType, Integer> transitionTimes; + public Map<NodeType, Integer> transitionTimes = buildTransitionTimeMap(0, 0); public long currentTimeInMillis = 0; public int maxPrematureCrashes = 0; public int minStorageNodesUp = 1; @@ -40,7 +39,6 @@ public class ClusterStateGenerator { public int maxInitProgressTimeMs = 5000; Params() { - this.transitionTimes = buildTransitionTimeMap(0, 0); } // FIXME de-dupe @@ -269,18 +267,40 @@ public class ClusterStateGenerator { final GroupAvailabilityCalculator calc = new GroupAvailabilityCalculator.Builder() .withMinNodeRatioPerGroup(params.minNodeRatioPerGroup) .withDistribution(params.cluster.getDistribution()) + .withNodesSafelySetToMaintenance(params.cluster.nodesSafelySetTo(State.MAINTENANCE)) .build(); - final Set<Integer> nodesToTakeDown = calc.nodesThatShouldBeDown(workingState); + GroupAvailabilityCalculator.Result result = calc.calculate(workingState); - for (Integer idx : nodesToTakeDown) { - final Node node = storageNode(idx); - final NodeState newState = new NodeState(NodeType.STORAGE, State.DOWN); - newState.setDescription("group node availability below configured threshold"); - workingState.setNodeState(node, newState); - nodeStateReasons.put(node, NodeStateReason.GROUP_IS_DOWN); + for (int index : result.nodesThatShouldBeMaintained()) { + setNewNodeState(index, NodeType.STORAGE, State.MAINTENANCE, + "too many safe maintenance nodes in group", NodeStateReason.GROUP_IN_MAINTENANCE, + workingState, nodeStateReasons); + + setNewNodeState(index, NodeType.DISTRIBUTOR, State.DOWN, + "too many safe maintenance nodes in group", NodeStateReason.GROUP_IN_MAINTENANCE, + workingState, nodeStateReasons); + } + + for (int index : result.nodesThatShouldBeDown()) { + setNewNodeState(index, NodeType.STORAGE, State.DOWN, + "group node availability below configured threshold", NodeStateReason.GROUP_IS_DOWN, + workingState, nodeStateReasons); } } + private static void setNewNodeState(int index, + NodeType nodeType, + State newState, + String description, + NodeStateReason nodeStateReason, + ClusterState workingState, + Map<Node, NodeStateReason> nodeStateReasons) { + final Node node = new Node(nodeType, index); + final NodeState newNodeState = new NodeState(nodeType, newState).setDescription(description); + workingState.setNodeState(node, newNodeState); + nodeStateReasons.put(node, nodeStateReason); + } + private static Node storageNode(int index) { return new Node(NodeType.STORAGE, index); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index 5e775116bf3..dad2b91d3cc 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -14,6 +14,9 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRe import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; import java.util.*; +import java.util.stream.Collectors; + +import static com.yahoo.vdslib.state.NodeState.ORCHESTRATOR_RESERVED_DESCRIPTION; public class ContentCluster { @@ -188,6 +191,26 @@ public class ContentCluster { return nodeStateChangeChecker.evaluateTransition(node, clusterState, condition, oldState, newState); } + /** Returns the indices of the nodes that have been safely set to the given state by the Orchestrator (best guess). */ + public List<Integer> nodesSafelySetTo(State state) { + switch (state) { + case MAINTENANCE: // Orchestrator's ALLOWED_TO_BE_DOWN + case DOWN: // Orchestrator's PERMANENTLY_DOWN + return clusterInfo.getStorageNodeInfo().stream() + .filter(storageNodeInfo -> { + NodeState userWantedState = storageNodeInfo.getUserWantedState(); + return userWantedState.getState() == state && + Objects.equals(userWantedState.getDescription(), ORCHESTRATOR_RESERVED_DESCRIPTION); + }) + .map(NodeInfo::getNodeIndex) + .collect(Collectors.toList()); + default: + // Note: There is no trace left if the Orchestrator set the state to UP, so that's handled + // like any other state: + return List.of(); + } + } + public void setMinStorageNodesUp(int minStorageNodesUp) { this.minStorageNodesUp = minStorageNodesUp; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculator.java index 686ef0dee6c..0afad4a0efe 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculator.java @@ -7,29 +7,37 @@ import com.yahoo.vdslib.distribution.Group; import com.yahoo.vdslib.distribution.GroupVisitor; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; -import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; -import com.yahoo.vdslib.state.State; -import java.util.Collections; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; class GroupAvailabilityCalculator { private final Distribution distribution; private final double minNodeRatioPerGroup; + private final int safeMaintenanceGroupThreshold; + private List<Integer> nodesSafelySetToMaintenance; private GroupAvailabilityCalculator(Distribution distribution, - double minNodeRatioPerGroup) { + double minNodeRatioPerGroup, + int safeMaintenanceGroupThreshold, + List<Integer> nodesSafelySetToMaintenance) { this.distribution = distribution; this.minNodeRatioPerGroup = minNodeRatioPerGroup; + this.safeMaintenanceGroupThreshold = safeMaintenanceGroupThreshold; + this.nodesSafelySetToMaintenance = nodesSafelySetToMaintenance; } public static class Builder { private Distribution distribution; private double minNodeRatioPerGroup = 1.0; + private int safeMaintenanceGroupThreshold = 2; + private List<Integer> nodesSafelySetToMaintenance = new ArrayList<>(); Builder withDistribution(Distribution distribution) { this.distribution = distribution; @@ -39,8 +47,23 @@ class GroupAvailabilityCalculator { this.minNodeRatioPerGroup = minRatio; return this; } + /** + * If the number of nodes safely set to maintenance is at least this number, the remaining + * nodes in the group will be set to maintenance (storage nodes) or down (distributors). + * + * <p>This feature is disabled if safeMaintenanceGroupThreshold is 0 (not default).</p> + */ + Builder withSafeMaintenanceGroupThreshold(int safeMaintenanceGroupThreshold) { + this.safeMaintenanceGroupThreshold = safeMaintenanceGroupThreshold; + return this; + } + Builder withNodesSafelySetToMaintenance(List<Integer> nodesSafelySetToMaintenance) { + this.nodesSafelySetToMaintenance.addAll(nodesSafelySetToMaintenance); + return this; + } GroupAvailabilityCalculator build() { - return new GroupAvailabilityCalculator(distribution, minNodeRatioPerGroup); + return new GroupAvailabilityCalculator(distribution, minNodeRatioPerGroup, + safeMaintenanceGroupThreshold, nodesSafelySetToMaintenance); } } @@ -49,11 +72,18 @@ class GroupAvailabilityCalculator { } private class InsufficientAvailabilityGroupVisitor implements GroupVisitor { + private final Set<Integer> implicitlyMaintained = new HashSet<>(); private final Set<Integer> implicitlyDown = new HashSet<>(); private final ClusterState clusterState; + private final Set<Integer> nodesSafelySetToMaintenance; + private final int safeMaintenanceGroupThreshold; - public InsufficientAvailabilityGroupVisitor(ClusterState clusterState) { + public InsufficientAvailabilityGroupVisitor(ClusterState clusterState, + List<Integer> nodesSafelySetToMaintenance, + int safeMaintenanceGroupThreshold) { this.clusterState = clusterState; + this.nodesSafelySetToMaintenance = Set.copyOf(nodesSafelySetToMaintenance); + this.safeMaintenanceGroupThreshold = safeMaintenanceGroupThreshold; } private boolean nodeIsAvailableInState(final int index, final String states) { @@ -75,6 +105,14 @@ class GroupAvailabilityCalculator { return g.getNodes().stream().filter(n -> nodeIsAvailableInState(n.index(), "ui")); } + private Stream<ConfiguredNode> candidateNodesForSettingMaintenance(Group g) { + // Most states should be set in maintenance, e.g. retirement may take a long time, + // so force maintenance to allow upgrades. + return g.getNodes().stream() + // "m" is NOT included since that would be a no-op. + .filter(n -> nodeIsAvailableInState(n.index(), "uird")); + } + private double computeGroupAvailability(Group g) { // TODO also look at distributors final long availableNodes = availableNodesIn(g).count(); @@ -83,22 +121,43 @@ class GroupAvailabilityCalculator { return availableNodes / (double)g.getNodes().size(); } + private int computeNodesSafelySetToMaintenance(Group group) { + Set<ConfiguredNode> nodesInGroupSafelySetToMaintenance = group.getNodes().stream() + .filter(configuredNode -> nodesSafelySetToMaintenance.contains(configuredNode.index())) + .collect(Collectors.toSet()); + + return nodesInGroupSafelySetToMaintenance.size(); + } + private void markAllAvailableGroupNodeIndicesAsDown(Group group) { candidateNodesForSettingDown(group).forEach(n -> implicitlyDown.add(n.index())); } + private void markAllAvailableGroupNodeIndicesAsMaintained(Group group) { + candidateNodesForSettingMaintenance(group).forEach(n -> implicitlyMaintained.add(n.index())); + } + @Override public boolean visitGroup(Group group) { if (group.isLeafGroup()) { - if (computeGroupAvailability(group) < minNodeRatioPerGroup) { + if (safeMaintenanceGroupThreshold > 0 && + computeNodesSafelySetToMaintenance(group) >= safeMaintenanceGroupThreshold) { + markAllAvailableGroupNodeIndicesAsMaintained(group); + } else if (computeGroupAvailability(group) < minNodeRatioPerGroup) { markAllAvailableGroupNodeIndicesAsDown(group); } } return true; } - Set<Integer> implicitlyDownNodeIndices() { - return implicitlyDown; + Result result() { + var intersection = new HashSet<>(implicitlyMaintained); + intersection.retainAll(implicitlyDown); + if (intersection.size() > 0) { + throw new IllegalStateException("Nodes implicitly both maintenance and down: " + intersection); + } + + return new Result(implicitlyMaintained, implicitlyDown); } } @@ -106,17 +165,37 @@ class GroupAvailabilityCalculator { return root.isLeafGroup(); } - public Set<Integer> nodesThatShouldBeDown(ClusterState state) { + public static class Result { + private final Set<Integer> shouldBeMaintained; + private final Set<Integer> shouldBeDown; + + public Result() { this(Set.of(), Set.of()); } + + public Result(Set<Integer> shouldBeMaintained, Set<Integer> shouldBeDown) { + this.shouldBeMaintained = Set.copyOf(shouldBeMaintained); + this.shouldBeDown = Set.copyOf(shouldBeDown); + } + + public Set<Integer> nodesThatShouldBeMaintained() { return shouldBeMaintained; } + public Set<Integer> nodesThatShouldBeDown() { return shouldBeDown; } + } + + public Result calculate(ClusterState state) { if (distribution == null) { // FIXME: for tests that don't set distribution properly! - return Collections.emptySet(); + return new Result(); } if (isFlatCluster(distribution.getRootGroup())) { // Implicit group takedown only applies to hierarchic cluster setups. - return new HashSet<>(); + return new Result(); } - InsufficientAvailabilityGroupVisitor visitor = new InsufficientAvailabilityGroupVisitor(state); + InsufficientAvailabilityGroupVisitor visitor = new InsufficientAvailabilityGroupVisitor( + state, nodesSafelySetToMaintenance, safeMaintenanceGroupThreshold); distribution.visitGroups(visitor); - return visitor.implicitlyDownNodeIndices(); + return visitor.result(); + } + + public Set<Integer> nodesThatShouldBeDown(ClusterState state) { + return calculate(state).nodesThatShouldBeDown(); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java index 3f550724cef..77ab8539219 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java @@ -7,7 +7,7 @@ public enum NodeStateReason { NODE_TOO_UNSTABLE, WITHIN_MAINTENANCE_GRACE_PERIOD, NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD, - FORCED_INTO_MAINTENANCE, + GROUP_IN_MAINTENANCE, GROUP_IS_DOWN, MAY_HAVE_MERGES_PENDING } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java index 08329c874b5..0ca8b010191 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java @@ -704,6 +704,34 @@ public class ClusterStateGeneratorTest { assertThat(state.toString(), equalTo("distributor:9 storage:9 .3.s:m .4.s:d .5.s:d")); } + @Test + public void group_nodes_are_marked_maintenance_if_group_availability_too_low_by_orchestrator() { + final ClusterFixture fixture = ClusterFixture + .forHierarchicCluster(DistributionBuilder.withGroups(3).eachWithNodeCount(3)) + .bringEntireClusterUp() + .proposeStorageNodeWantedState(4, State.MAINTENANCE, NodeState.ORCHESTRATOR_RESERVED_DESCRIPTION) + .proposeStorageNodeWantedState(5, State.MAINTENANCE, NodeState.ORCHESTRATOR_RESERVED_DESCRIPTION); + final ClusterStateGenerator.Params params = fixture.generatorParams(); + + // Both node 4 & 5 are in maintenance by Orchestrator, which will force the other nodes in the + // group to maintenance (node 3). + final AnnotatedClusterState state = ClusterStateGenerator.generatedStateFrom(params); + assertThat(state.toString(), equalTo("distributor:9 .3.s:d storage:9 .3.s:m .4.s:m .5.s:m")); + } + + @Test + public void group_nodes_are_not_marked_maintenance_if_group_availability_high_by_orchestrator() { + final ClusterFixture fixture = ClusterFixture + .forHierarchicCluster(DistributionBuilder.withGroups(3).eachWithNodeCount(3)) + .bringEntireClusterUp() + .proposeStorageNodeWantedState(4, State.MAINTENANCE, NodeState.ORCHESTRATOR_RESERVED_DESCRIPTION); + final ClusterStateGenerator.Params params = fixture.generatorParams(); + + // Node 4 is in maintenance by Orchestrator, which is not sufficient to force group into maintenance. + final AnnotatedClusterState state = ClusterStateGenerator.generatedStateFrom(params); + assertThat(state.toString(), equalTo("distributor:9 storage:9 .4.s:m")); + } + /** * Cluster-wide distribution bit count cannot be higher than the lowest split bit * count reported by the set of storage nodes. This is because the distribution bit diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculatorTest.java index 07bd18c3667..f77a68e0fda 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAvailabilityCalculatorTest.java @@ -7,6 +7,7 @@ import org.junit.Test; import java.text.ParseException; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import static org.hamcrest.CoreMatchers.equalTo; @@ -182,4 +183,44 @@ public class GroupAvailabilityCalculatorTest { "distributor:8 storage:8 .0.s:d .1.s:d .2.s:d .4.s:d .5.s:d .6.s:d")), equalTo(emptySet())); } + @Test + public void one_safe_maintenance_node_does_not_take_down_group() { + // 2 groups of 5 nodes each. Set node #5 safely in maintenance (1st node in last group). + // Since the minimum number of nodes that can safely be set to maintenance before taking + // the whole group down is 2, the whole group should NOT be taken down. + + DistributionBuilder.GroupBuilder groupBuilder = DistributionBuilder.withGroups(2).eachWithNodeCount(5); + GroupAvailabilityCalculator calculator = GroupAvailabilityCalculator.builder() + .withDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)) + .withMinNodeRatioPerGroup(0) + .withSafeMaintenanceGroupThreshold(2) + .withNodesSafelySetToMaintenance(List.of(5)) + .build(); + + GroupAvailabilityCalculator.Result result = calculator + .calculate(clusterState("distributor:10 storage:10 .5.s:m .6.s:m .8.s:r .9.s:d")); + assertThat(result.nodesThatShouldBeMaintained(), equalTo(indices())); + assertThat(result.nodesThatShouldBeDown(), equalTo(indices())); + } + + @Test + public void two_safe_maintenance_nodes_takes_down_group() { + // 2 groups of 5 nodes each. Set nodes #5 and #6 safely in maintenance (1st and 2nd nodes + // in last group, respectively). Since the minimum number of nodes that can safely be set to + // maintenance before taking the whole group down is 2, the whole group should be taken down. + + DistributionBuilder.GroupBuilder groupBuilder = DistributionBuilder.withGroups(2).eachWithNodeCount(5); + GroupAvailabilityCalculator calculator = GroupAvailabilityCalculator.builder() + .withDistribution(DistributionBuilder.forHierarchicCluster(groupBuilder)) + .withMinNodeRatioPerGroup(0) + .withSafeMaintenanceGroupThreshold(2) + .withNodesSafelySetToMaintenance(List.of(5, 6)) + .build(); + + GroupAvailabilityCalculator.Result result = calculator + .calculate(clusterState("distributor:10 storage:10 .5.s:m .6.s:m .8.s:r .9.s:d")); + assertThat(result.nodesThatShouldBeMaintained(), equalTo(indices(7, 8, 9))); + assertThat(result.nodesThatShouldBeDown(), equalTo(indices())); + } + } diff --git a/clustercontroller-reindexer/pom.xml b/clustercontroller-reindexer/pom.xml index 172bff6fdb4..e0577c438b5 100644 --- a/clustercontroller-reindexer/pom.xml +++ b/clustercontroller-reindexer/pom.xml @@ -63,6 +63,12 @@ <artifactId>junit-jupiter-engine</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3057be6251c..d020079bb17 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -31,8 +31,6 @@ public interface ModelContext { ApplicationPackage applicationPackage(); Optional<Model> previousModel(); Optional<ApplicationPackage> permanentApplicationPackage(); - // TODO: Remove after 7.338 has been released - default Optional<HostProvisioner> hostProvisioner() { return Optional.of(getHostProvisioner()); } HostProvisioner getHostProvisioner(); Provisioned provisioned(); DeployLogger deployLogger(); @@ -70,7 +68,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "jonmv"}) default double reindexerWindowSizeIncrement() { return 0.2; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"geirst"}, comment = "Remove when 7.342 is no longer in use") default boolean useDirectStorageApiRpc() { return true; } + @ModelFeatureFlag(owners = {"geirst"}, removeAfter = "7.342") default boolean useDirectStorageApiRpc() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } @@ -84,6 +82,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"musum", "mpolden"}, comment = "Revisit in February 2021") default boolean reconfigurableZookeeperServer() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default boolean enableJdiscConnectionLog() { return false; } + @ModelFeatureFlag(owners = {"bjorncs", "tokle", "baldersheim"}) default boolean enableZstdCompressionAccessLog() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java deleted file mode 100644 index 0937b8b77ec..00000000000 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.api; - -public class TlsSecrets { - public static final TlsSecrets MISSING = new TlsSecrets(); - - private final String certificate; - private final String key; - - private TlsSecrets() { - this(null, null); - } - - public TlsSecrets(String certificate, String key) { - this.certificate = certificate; - this.key = key; - } - - public TlsSecrets(EndpointCertificateSecrets endpointCertificateSecrets) { - this.certificate = endpointCertificateSecrets.certificate(); - this.key = endpointCertificateSecrets.key(); - } - - public String certificate() { - return certificate; - } - - public String key() { - return key; - } - - public boolean isMissing() { - return this == MISSING; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index ad491b3ca0c..a4b01259a72 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -15,7 +15,7 @@ public class LogserverContainer extends Container { public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) { super(parent, featureFlags, "" + 0, 0, isHostedVespa); - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true)); + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true, false)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 6379ddcec95..e2df04ebbf3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -57,7 +57,8 @@ public class ClusterControllerContainer extends Container implements CLUSTERCONTROLLER_BUNDLE); addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, "controller", - deployState.isHosted())); + deployState.isHosted(), + deployState.featureFlags().enableZstdCompressionAccessLog())); // TODO: Why are bundles added here instead of in the cluster? addFileBundle("clustercontroller-apps"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index f4805e7ac15..23a860e2b67 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -530,8 +530,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> if (containerSearch != null) containerSearch.connectSearchClusters(clusterMap); } - public void addDefaultSearchAccessLog() { - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, getName(), isHostedVespa)); + public void addDefaultSearchAccessLog(DeployState deployState) { + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, getName(), isHostedVespa, deployState.featureFlags().enableZstdCompressionAccessLog())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index e75812a9ad6..6e197915a64 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.core.AccessLogConfig; +import com.yahoo.container.core.AccessLogConfig.FileHandler.CompressionFormat; import com.yahoo.container.logging.VespaAccessLog; import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.osgi.provider.model.ComponentModel; @@ -19,13 +20,15 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final Boolean compression; private final boolean isHostedVespa; private final String symlinkName; + private final boolean enableZstdCompression; - public AccessLogComponent(AccessLogType logType, String clusterName, boolean isHostedVespa) + public AccessLogComponent(AccessLogType logType, String clusterName, boolean isHostedVespa, boolean enableZstdCompression) { this(logType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, - capitalize(logType.name()) + "." + clusterName); + capitalize(logType.name()) + "." + clusterName, + enableZstdCompression); } private static String capitalize(String name) { @@ -37,7 +40,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL String rotationInterval, Boolean compressOnRotation, boolean isHostedVespa, - String symlinkName) + String symlinkName, + boolean enableZstdCompression) { super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; @@ -45,6 +49,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; + this.enableZstdCompression = enableZstdCompression; if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); @@ -79,6 +84,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } else if (isHostedVespa) { builder.compressOnRotation(true); } + if (enableZstdCompression) { + builder.compressionFormat(CompressionFormat.Enum.ZSTD); + } return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index a29c54f951b..5b87dc57306 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -59,7 +59,8 @@ public class AccessLogBuilder { rotationInterval(spec), compressOnRotation(spec), isHostedVespa, - symlinkName(spec)); + symlinkName(spec), + deployState.featureFlags().enableZstdCompressionAccessLog()); } private String symlinkName(Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 15a37813150..d650b10a910 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -346,7 +346,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) - cluster.addDefaultSearchAccessLog(); + cluster.addDefaultSearchAccessLog(deployState); // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 98cbd363bca..f8469aa6fa1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -50,11 +50,6 @@ public class MockModelContext implements ModelContext { } @Override - public Optional<HostProvisioner> hostProvisioner() { - return Optional.empty(); - } - - @Override public HostProvisioner getHostProvisioner() { return DeployState.getDefaultModelHostProvisioner(applicationPackage); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 33f9d715801..e3e0edd7896 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -141,11 +141,6 @@ public class VespaModelFactoryTest { } @Override - public Optional<HostProvisioner> hostProvisioner() { - return Optional.of(provisionerToOverride); - } - - @Override public HostProvisioner getHostProvisioner() { return provisionerToOverride; } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java index 7f7357ea77f..d4c52b97f45 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -64,6 +64,11 @@ public class ClusterResources { return new ClusterResources(nodes, groups, nodeResources.justNumbers()); } + /** Returns the standard cost of these resources, in dollars per hour */ + public double cost() { + return nodes * nodeResources.cost(); + } + @Override public boolean equals(Object o) { if (o == this) return true; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index 49a61eb8dee..ec4523c4ba0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -2,25 +2,12 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.TenantListener; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; import java.time.Clock; -import java.util.Optional; -import java.util.concurrent.ExecutorService; /** * Interface representing all global config server components used within the config server. @@ -29,22 +16,12 @@ import java.util.concurrent.ExecutorService; */ public interface GlobalComponentRegistry { - Curator getCurator(); - ConfigCurator getConfigCurator(); - Metrics getMetrics(); - SessionPreparer getSessionPreparer(); ConfigserverConfig getConfigserverConfig(); TenantListener getTenantListener(); ReloadListener getReloadListener(); ConfigDefinitionRepo getStaticConfigDefinitionRepo(); - PermanentApplicationPackage getPermanentApplicationPackage(); ModelFactoryRegistry getModelFactoryRegistry(); - Optional<Provisioner> getHostProvisioner(); Zone getZone(); Clock getClock(); ConfigServerDB getConfigServerDB(); - StripedExecutor<TenantName> getZkWatcherExecutor(); - FlagSource getFlagSource(); - ExecutorService getZkCacheExecutor(); - SecretStore getSecretStore(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index 5760eae866c..433e0971c61 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -3,30 +3,13 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.rpc.RpcServer; -import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.TenantListener; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; import java.time.Clock; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Registry containing all the "static"/"global" components in a config server in one place. @@ -35,69 +18,30 @@ import java.util.concurrent.Executors; */ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry { - private final Curator curator; - private final ConfigCurator configCurator; - private final Metrics metrics; private final ModelFactoryRegistry modelFactoryRegistry; - private final SessionPreparer sessionPreparer; private final RpcServer rpcServer; private final ConfigserverConfig configserverConfig; private final ConfigDefinitionRepo staticConfigDefinitionRepo; - private final PermanentApplicationPackage permanentApplicationPackage; - private final Optional<Provisioner> hostProvisioner; private final Zone zone; private final ConfigServerDB configServerDB; - private final FlagSource flagSource; - private final SecretStore secretStore; - private final StripedExecutor<TenantName> zkWatcherExecutor; - private final ExecutorService zkCacheExecutor; - private final HostRegistry hostRegistry; @SuppressWarnings("WeakerAccess") @Inject - public InjectedGlobalComponentRegistry(Curator curator, - ConfigCurator configCurator, - Metrics metrics, - ModelFactoryRegistry modelFactoryRegistry, - SessionPreparer sessionPreparer, + public InjectedGlobalComponentRegistry(ModelFactoryRegistry modelFactoryRegistry, RpcServer rpcServer, ConfigserverConfig configserverConfig, ConfigDefinitionRepo staticConfigDefinitionRepo, - PermanentApplicationPackage permanentApplicationPackage, - HostProvisionerProvider hostProvisionerProvider, Zone zone, - ConfigServerDB configServerDB, - FlagSource flagSource, - SecretStore secretStore, - HostRegistry hostRegistry) { - this.curator = curator; - this.configCurator = configCurator; - this.metrics = metrics; + ConfigServerDB configServerDB) { this.modelFactoryRegistry = modelFactoryRegistry; - this.sessionPreparer = sessionPreparer; this.rpcServer = rpcServer; this.configserverConfig = configserverConfig; this.staticConfigDefinitionRepo = staticConfigDefinitionRepo; - this.permanentApplicationPackage = permanentApplicationPackage; - this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); this.zone = zone; this.configServerDB = configServerDB; - this.flagSource = flagSource; - this.secretStore = secretStore; - this.zkWatcherExecutor = new StripedExecutor<>(); - this.zkCacheExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(TenantRepository.class.getName())); - this.hostRegistry = hostRegistry; } @Override - public Curator getCurator() { return curator; } - @Override - public ConfigCurator getConfigCurator() { return configCurator; } - @Override - public Metrics getMetrics() { return metrics; } - @Override - public SessionPreparer getSessionPreparer() { return sessionPreparer; } - @Override public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } @Override public TenantListener getTenantListener() { return rpcServer; } @@ -106,16 +50,9 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry @Override public ConfigDefinitionRepo getStaticConfigDefinitionRepo() { return staticConfigDefinitionRepo; } @Override - public PermanentApplicationPackage getPermanentApplicationPackage() { return permanentApplicationPackage; } - @Override public ModelFactoryRegistry getModelFactoryRegistry() { return modelFactoryRegistry; } @Override - public Optional<Provisioner> getHostProvisioner() { - return hostProvisioner; - } - - @Override public Zone getZone() { return zone; } @@ -126,22 +63,4 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry @Override public ConfigServerDB getConfigServerDB() { return configServerDB; } - @Override - public StripedExecutor<TenantName> getZkWatcherExecutor() { - return zkWatcherExecutor; - } - - @Override - public FlagSource getFlagSource() { return flagSource; } - - @Override - public ExecutorService getZkCacheExecutor() { - return zkCacheExecutor; - } - - @Override - public SecretStore getSecretStore() { - return secretStore; - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index c4908b8827b..2b9488b7fc7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; @@ -87,12 +88,15 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica } // For testing only - public static TenantApplications create(GlobalComponentRegistry componentRegistry, HostRegistry hostRegistry, TenantName tenantName) { + public static TenantApplications create(GlobalComponentRegistry componentRegistry, + HostRegistry hostRegistry, + TenantName tenantName, + Curator curator) { return new TenantApplications(tenantName, - componentRegistry.getCurator(), - componentRegistry.getZkWatcherExecutor(), - componentRegistry.getZkCacheExecutor(), - componentRegistry.getMetrics(), + curator, + new StripedExecutor<>(new InThreadExecutorService()), + new InThreadExecutorService(), + Metrics.createTestMetrics(), componentRegistry.getReloadListener(), componentRegistry.getConfigserverConfig(), hostRegistry, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 997ea32ff0b..596f9c0ce98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -164,6 +164,7 @@ public class ModelContextImpl implements ModelContext { private final double feedConcurrency; private final boolean reconfigurableZookeeperServer; private final boolean enableJdiscConnectionLog; + private final boolean enableZstdCompressionAccessLog; public FeatureFlags(FlagSource source, ApplicationId appId) { this.enableAutomaticReindexing = flagValue(source, appId, Flags.ENABLE_AUTOMATIC_REINDEXING); @@ -183,6 +184,7 @@ public class ModelContextImpl implements ModelContext { this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.reconfigurableZookeeperServer = flagValue(source, appId, Flags.RECONFIGURABLE_ZOOKEEPER_SERVER_FOR_CLUSTER_CONTROLLER); this.enableJdiscConnectionLog = flagValue(source, appId, Flags.ENABLE_JDISC_CONNECTION_LOG); + this.enableZstdCompressionAccessLog = flagValue(source, appId, Flags.ENABLE_ZSTD_COMPRESSION_ACCESS_LOG); } @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @@ -202,6 +204,7 @@ public class ModelContextImpl implements ModelContext { @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean enableJdiscConnectionLog() { return enableJdiscConnectionLog; } + @Override public boolean enableZstdCompressionAccessLog() { return enableZstdCompressionAccessLog; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) @@ -230,7 +233,7 @@ public class ModelContextImpl implements ModelContext { private final Optional<ApplicationRoles> applicationRoles; private final Quota quota; - private final String jvmGCOPtions; + private final String jvmGcOptions; public Properties(ApplicationId applicationId, ConfigserverConfig configserverConfig, @@ -260,7 +263,7 @@ public class ModelContextImpl implements ModelContext { this.applicationRoles = applicationRoles; this.quota = maybeQuota.orElseGet(Quota::unlimited); - jvmGCOPtions = flagValue(flagSource, applicationId, PermanentFlags.JVM_GC_OPTIONS); + jvmGcOptions = flagValue(flagSource, applicationId, PermanentFlags.JVM_GC_OPTIONS); } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -315,7 +318,7 @@ public class ModelContextImpl implements ModelContext { @Override public Quota quota() { return quota; } - @Override public String jvmGCOptions() { return jvmGCOPtions; } + @Override public String jvmGCOptions() { return jvmGcOptions; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index fa058514d17..f779cad245a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -65,21 +65,27 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { long applicationGeneration, SessionZooKeeperClient zkClient, Optional<ApplicationSet> currentActiveApplicationSet, - GlobalComponentRegistry globalComponentRegistry) { + GlobalComponentRegistry globalComponentRegistry, + Curator curator, + Metrics metrics, + PermanentApplicationPackage permanentApplicationPackage, + FlagSource flagSource, + SecretStore secretStore, + HostProvisionerProvider hostProvisionerProvider) { super(globalComponentRegistry.getModelFactoryRegistry(), globalComponentRegistry.getConfigserverConfig(), globalComponentRegistry.getZone(), - HostProvisionerProvider.from(globalComponentRegistry.getHostProvisioner())); + hostProvisionerProvider); this.tenant = tenant; this.applicationGeneration = applicationGeneration; this.zkClient = zkClient; this.currentActiveApplicationSet = currentActiveApplicationSet; - this.permanentApplicationPackage = globalComponentRegistry.getPermanentApplicationPackage(); + this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = globalComponentRegistry.getStaticConfigDefinitionRepo(); - this.metrics = globalComponentRegistry.getMetrics(); - this.curator = globalComponentRegistry.getCurator(); - this.flagSource = globalComponentRegistry.getFlagSource(); - this.secretStore = globalComponentRegistry.getSecretStore(); + this.metrics = metrics; + this.curator = curator; + this.flagSource = flagSource; + this.secretStore = secretStore; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 13c21a065ff..729f1df31c1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.provision; import com.yahoo.cloud.config.ConfigserverConfig; @@ -42,18 +42,15 @@ public class HostProvisionerProvider { } // for testing - public static HostProvisionerProvider withProvisioner(Provisioner provisioner) { - ComponentRegistry<Provisioner> registry = new ComponentRegistry<>(); - registry.register(ComponentId.createAnonymousComponentId("foobar"), provisioner); - return new HostProvisionerProvider(registry, new ConfigserverConfig(new ConfigserverConfig.Builder().hostedVespa(true))); + public static HostProvisionerProvider withProvisioner(Provisioner provisioner, boolean hostedVespa) { + return withProvisioner(provisioner, new ConfigserverConfig(new ConfigserverConfig.Builder().hostedVespa(hostedVespa))); } - /** Creates either an empty provider or a provider having the given provisioner */ - public static HostProvisionerProvider from(Optional<Provisioner> provisioner) { - if (provisioner.isPresent()) - return withProvisioner(provisioner.get()); - else - return empty(); + // for testing + public static HostProvisionerProvider withProvisioner(Provisioner provisioner, ConfigserverConfig config) { + ComponentRegistry<Provisioner> registry = new ComponentRegistry<>(); + registry.register(ComponentId.createAnonymousComponentId("foobar"), provisioner); + return new HostProvisionerProvider(registry, config); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 139081fde00..78071cbf89e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -37,7 +37,6 @@ public final class PrepareParams { static final String VERBOSE_PARAM_NAME = "verbose"; static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; - static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository"; static final String ATHENZ_DOMAIN = "athenzDomain"; @@ -55,7 +54,6 @@ public final class PrepareParams { private final boolean force; private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; - private final Optional<String> tlsSecretsKeyName; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<DockerImage> dockerImageRepository; private final Optional<AthenzDomain> athenzDomain; @@ -64,7 +62,7 @@ public final class PrepareParams { private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, - List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, + List<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) { @@ -76,7 +74,6 @@ public final class PrepareParams { this.isBootstrap = isBootstrap; this.vespaVersion = vespaVersion; this.containerEndpoints = containerEndpoints; - this.tlsSecretsKeyName = tlsSecretsKeyName; this.endpointCertificateMetadata = endpointCertificateMetadata; this.dockerImageRepository = dockerImageRepository; this.athenzDomain = athenzDomain; @@ -96,7 +93,6 @@ public final class PrepareParams { private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = null; - private Optional<String> tlsSecretsKeyName = Optional.empty(); private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); private Optional<DockerImage> dockerImageRepository = Optional.empty(); private Optional<AthenzDomain> athenzDomain = Optional.empty(); @@ -156,12 +152,6 @@ public final class PrepareParams { return this; } - public Builder tlsSecretsKeyName(String tlsSecretsKeyName) { - this.tlsSecretsKeyName = Optional.ofNullable(tlsSecretsKeyName) - .filter(s -> ! s.isEmpty()); - return this; - } - public Builder endpointCertificateMetadata(String serialized) { this.endpointCertificateMetadata = (serialized == null) ? Optional.empty() @@ -210,7 +200,7 @@ public final class PrepareParams { public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, + verbose, isBootstrap, vespaVersion, containerEndpoints, endpointCertificateMetadata, dockerImageRepository, athenzDomain, applicationRoles, quota, force); } @@ -224,7 +214,6 @@ public final class PrepareParams { .applicationId(createApplicationId(request, tenant)) .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) - .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY)) .athenzDomain(request.getProperty(ATHENZ_DOMAIN)) @@ -284,10 +273,6 @@ public final class PrepareParams { return timeoutBudget; } - public Optional<String> tlsSecretsKeyName() { - return tlsSecretsKeyName; - } - public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { return endpointCertificateMetadata; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 5609de68391..b29259e22d4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.session; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; @@ -80,7 +79,6 @@ public class SessionPreparer { private final SecretStore secretStore; private final FlagSource flagSource; - @Inject public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry, FileDistributionFactory fileDistributionFactory, HostProvisionerProvider hostProvisionerProvider, @@ -183,8 +181,7 @@ public class SessionPreparer { this.containerEndpointsCache = new ContainerEndpointsCache(tenantPath, curator); this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); EndpointCertificateRetriever endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); - this.endpointCertificateMetadata = params.endpointCertificateMetadata() - .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); + this.endpointCertificateMetadata = params.endpointCertificateMetadata(); Optional<EndpointCertificateSecrets> endpointCertificateSecrets = endpointCertificateMetadata .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 592198cbbef..041768ec41e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -11,6 +12,7 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.io.IOUtils; import com.yahoo.lang.SettableOptional; import com.yahoo.path.Path; @@ -20,6 +22,7 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; @@ -27,11 +30,13 @@ import com.yahoo.vespa.config.server.filedistribution.FileDirectory; import com.yahoo.vespa.config.server.modelfactory.ActivatedModelsBuilder; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.FlagSource; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -54,6 +59,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -81,34 +87,55 @@ public class SessionRepository { private final Clock clock; private final Curator curator; private final Executor zkWatcherExecutor; + private final PermanentApplicationPackage permanentApplicationPackage; + private final FlagSource flagSource; private final TenantFileSystemDirs tenantFileSystemDirs; - private final MetricUpdater metrics; + private final Metrics metrics; + private final MetricUpdater metricUpdater; private final Curator.DirectoryCache directoryCache; private final TenantApplications applicationRepo; private final SessionPreparer sessionPreparer; private final Path sessionsPath; private final TenantName tenantName; private final GlobalComponentRegistry componentRegistry; + private final ConfigCurator configCurator; private final SessionCounter sessionCounter; + private final SecretStore secretStore; + private final HostProvisionerProvider hostProvisionerProvider; public SessionRepository(TenantName tenantName, GlobalComponentRegistry componentRegistry, TenantApplications applicationRepo, - SessionPreparer sessionPreparer) { + SessionPreparer sessionPreparer, + Curator curator, + Metrics metrics, + StripedExecutor<TenantName> zkWatcherExecutor, + PermanentApplicationPackage permanentApplicationPackage, + FlagSource flagSource, + ExecutorService zkCacheExecutor, + SecretStore secretStore, + HostProvisionerProvider hostProvisionerProvider) { this.tenantName = tenantName; this.componentRegistry = componentRegistry; - sessionCounter = new SessionCounter(componentRegistry.getConfigCurator(), tenantName); + this.configCurator = ConfigCurator.create(curator); + sessionCounter = new SessionCounter(configCurator, tenantName); this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.clock = componentRegistry.getClock(); - this.curator = componentRegistry.getCurator(); + this.curator = curator; this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); - this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); + this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); + this.permanentApplicationPackage = permanentApplicationPackage; + this.flagSource = flagSource; this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); this.applicationRepo = applicationRepo; this.sessionPreparer = sessionPreparer; - this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + this.metrics = metrics; + this.metricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + this.secretStore = secretStore; + this.hostProvisionerProvider = hostProvisionerProvider; + loadSessions(); // Needs to be done before creating cache below - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor); this.directoryCache.addListener(this::childEvent); this.directoryCache.start(); } @@ -348,7 +375,7 @@ public class SessionRepository { SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); remoteSessionCache.remove(sessionId); - metrics.incRemovedSessions(); + metricUpdater.incRemovedSessions(); } private void loadSessionIfActive(RemoteSession session) { @@ -425,7 +452,13 @@ public class SessionRepository { session.getSessionId(), sessionZooKeeperClient, previousApplicationSet, - componentRegistry); + componentRegistry, + curator, + metrics, + permanentApplicationPackage, + flagSource, + secretStore, + hostProvisionerProvider); // Read hosts allocated on the config server instance which created this SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>(applicationPackage.getAllocatedHosts()); @@ -443,10 +476,10 @@ public class SessionRepository { for (Session session : remoteSessionCache.values()) { sessionMetrics.add(session.getStatus()); } - metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); - metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); - metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); - metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); + metricUpdater.setNewSessions(sessionMetrics.count(Session.Status.NEW)); + metricUpdater.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); + metricUpdater.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); + metricUpdater.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); }); } @@ -507,7 +540,7 @@ public class SessionRepository { private void ensureSessionPathDoesNotExist(long sessionId) { Path sessionPath = getSessionPath(sessionId); - if (componentRegistry.getConfigCurator().exists(sessionPath.getAbsolute())) { + if (configCurator.exists(sessionPath.getAbsolute())) { throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); } } @@ -677,7 +710,7 @@ public class SessionRepository { private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { String serverId = componentRegistry.getConfigserverConfig().serverId(); - return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), tenantName, sessionId, serverId); + return new SessionZooKeeperClient(curator, configCurator, tenantName, sessionId, serverId); } private File getAndValidateExistingSessionAppDir(long sessionId) { @@ -697,7 +730,7 @@ public class SessionRepository { if (sessionStateWatcher == null) { Curator.FileCache fileCache = curator.createFileCache(getSessionStatePath(sessionId).getAbsolute(), false); fileCache.addListener(this::nodeChanged); - sessionStateWatchers.put(sessionId, new SessionStateWatcher(fileCache, remoteSession, metrics, zkWatcherExecutor, this)); + sessionStateWatchers.put(sessionId, new SessionStateWatcher(fileCache, remoteSession, metricUpdater, zkWatcherExecutor, this)); } else { sessionStateWatcher.updateRemoteSession(remoteSession); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java index f3240a62133..13c88ece6d7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; /** * (de)serializes endpoint certificate metadata @@ -30,26 +30,13 @@ public class EndpointCertificateMetadataSerializer { } public static EndpointCertificateMetadata fromSlime(Inspector inspector) { - switch (inspector.type()) { - case STRING: // TODO: Remove once all are transmitted and stored as JSON - return new EndpointCertificateMetadata( - inspector.asString() + "-key", - inspector.asString() + "-cert", - 0 - ); - case OBJECT: - return new EndpointCertificateMetadata( - inspector.field(keyNameField).asString(), - inspector.field(certNameField).asString(), - Math.toIntExact(inspector.field(versionField).asLong()) - ); - - default: - throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); + if (inspector.type() == Type.OBJECT) { + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()) + ); } - } - - public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) { - return fromSlime(new Slime().setString(tlsSecretsKeys)); + throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java index 8e51ac424f9..415f0a41441 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -37,7 +37,7 @@ public class EndpointCertificateMetadataStore { EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get()); return Optional.of(endpointCertificateMetadata); } catch (Exception e) { - throw new RuntimeException("Error reading TLS secret key of " + application, e); + throw new RuntimeException("Error reading endpoint certificate metadata for " + application, e); } } @@ -48,17 +48,17 @@ public class EndpointCertificateMetadataStore { EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject()); curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime)); } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); + throw new RuntimeException("Could not write endpoint certificate metadata for " + application, e); } } - /** Returns a transaction which deletes these tls secrets key if they exist */ + /** Returns a transaction which deletes endpoint certificate metadata if it exists */ public CuratorTransaction delete(ApplicationId application) { if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator); return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator); } - /** Returns the path storing the tls secrets key for an application */ + /** Returns the path storing the endpoint certificate metadata for an application */ private Path endpointCertificateMetadataPathOf(ApplicationId application) { return path.append(application.serializedForm()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index ce382e8698e..7daf56df487 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -6,20 +6,28 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.flags.FlagSource; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.state.ConnectionState; @@ -82,10 +90,14 @@ public class TenantRepository { private final HostRegistry hostRegistry; private final List<TenantListener> tenantListeners = Collections.synchronizedList(new ArrayList<>()); private final Curator curator; - + private final Metrics metrics; private final MetricUpdater metricUpdater; private final ExecutorService zkCacheExecutor; private final StripedExecutor<TenantName> zkWatcherExecutor; + private final FileDistributionFactory fileDistributionFactory; + private final FlagSource flagSource; + private final SecretStore secretStore; + private final HostProvisionerProvider hostProvisionerProvider; private final ExecutorService bootstrapExecutor; private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications")); @@ -97,17 +109,51 @@ public class TenantRepository { * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} */ @Inject - public TenantRepository(GlobalComponentRegistry componentRegistry, HostRegistry hostRegistry) { + public TenantRepository(GlobalComponentRegistry componentRegistry, + HostRegistry hostRegistry, + Curator curator, + Metrics metrics, + FlagSource flagSource, + SecretStore secretStore, + HostProvisionerProvider hostProvisionerProvider) { + this(componentRegistry, + hostRegistry, + curator, + metrics, + new StripedExecutor<>(), + new FileDistributionFactory(componentRegistry.getConfigserverConfig()), + flagSource, + Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(TenantRepository.class.getName())), + secretStore, + hostProvisionerProvider); + } + + public TenantRepository(GlobalComponentRegistry componentRegistry, + HostRegistry hostRegistry, + Curator curator, + Metrics metrics, + StripedExecutor<TenantName> zkWatcherExecutor, + FileDistributionFactory fileDistributionFactory, + FlagSource flagSource, + ExecutorService zkCacheExecutor, + SecretStore secretStore, + HostProvisionerProvider hostProvisionerProvider) { this.componentRegistry = componentRegistry; this.hostRegistry = hostRegistry; ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), new DaemonThreadFactory("bootstrap tenants")); - this.curator = componentRegistry.getCurator(); - metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); + this.curator = curator; + this.metrics = metrics; + metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(componentRegistry.getTenantListener()); - this.zkCacheExecutor = componentRegistry.getZkCacheExecutor(); - this.zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); + this.zkCacheExecutor = zkCacheExecutor; + this.zkWatcherExecutor = zkWatcherExecutor; + this.fileDistributionFactory = fileDistributionFactory; + this.flagSource = flagSource; + this.secretStore = secretStore; + this.hostProvisionerProvider = hostProvisionerProvider; + curator.framework().getConnectionStateListenable().addListener(this::stateChanged); curator.create(tenantsPath); @@ -226,18 +272,37 @@ public class TenantRepository { TenantApplications applicationRepo = new TenantApplications(tenantName, curator, - componentRegistry.getZkWatcherExecutor(), - componentRegistry.getZkCacheExecutor(), - componentRegistry.getMetrics(), + zkWatcherExecutor, + zkCacheExecutor, + metrics, componentRegistry.getReloadListener(), componentRegistry.getConfigserverConfig(), hostRegistry, new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName), componentRegistry.getClock()); + PermanentApplicationPackage permanentApplicationPackage = new PermanentApplicationPackage(componentRegistry.getConfigserverConfig()); + SessionPreparer sessionPreparer = new SessionPreparer(componentRegistry.getModelFactoryRegistry(), + fileDistributionFactory, + hostProvisionerProvider, + permanentApplicationPackage, + componentRegistry.getConfigserverConfig(), + componentRegistry.getStaticConfigDefinitionRepo(), + curator, + componentRegistry.getZone(), + flagSource, + secretStore); SessionRepository sessionRepository = new SessionRepository(tenantName, componentRegistry, applicationRepo, - componentRegistry.getSessionPreparer()); + sessionPreparer, + curator, + metrics, + zkWatcherExecutor, + permanentApplicationPackage, + flagSource, + zkCacheExecutor, + secretStore, + hostProvisionerProvider); log.log(Level.INFO, "Adding tenant '" + tenantName + "'" + ", created " + created); Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, applicationRepo, created); notifyNewTenant(tenant); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index 20ac4b65c64..24553704862 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -239,11 +239,11 @@ public class ConfigCurator { curator.framework().checkExists().forPath("/dummy"); } catch (Exception e) { - log.log(Level.SEVERE, "Unable to contact ZooKeeper on " + curator.connectionSpec() + - ". Please verify for all configserver nodes that " + - "VESPA_CONFIGSERVERS points to the correct configserver(s), " + - "the same configserver(s) as in services.xml, and that they are started. " + - "Check the log(s) for configserver errors. Aborting.", e); + log.log(Level.SEVERE, "Unable to connect to ZooKeeper on " + curator.connectionSpec() + + ". Please verify that VESPA_CONFIGSERVERS points to the correct configserver(s) " + + "on all config server nodes and are the same config server(s) as in services.xml, " + + "and that they are started. " + + "Check the log(s) for config server errors. Aborting.", e); } } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 00c65644e47..53cfa24b2ac 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -18,7 +18,6 @@ <component id="com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.SuperModelGenerationCounter" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.SuperModelManager" bundle="configserver" /> - <component id="com.yahoo.vespa.config.server.session.SessionPreparer" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.SuperModelRequestHandler" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.StaticConfigDefinitionRepo" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.provision.HostProvisionerProvider" bundle="configserver" /> @@ -32,7 +31,6 @@ <component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.zookeeper.ConfigCurator" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker" bundle="configserver" /> - <component id="com.yahoo.container.jdisc.metric.state.StateMetricConsumerFactory" bundle="container-disc" /> <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> <component id="com.yahoo.vespa.config.server.application.ConfigConvergenceChecker" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index b45569c95b8..36e1698d69f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -32,7 +32,7 @@ import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -42,6 +42,7 @@ import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; @@ -121,14 +122,17 @@ public class ApplicationRepositoryTest { .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); - InMemoryFlagSource flagSource = new InMemoryFlagSource(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) .configServerConfig(configserverConfig) - .flagSource(flagSource) .clock(clock) .build(); - tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + InMemoryFlagSource flagSource = new InMemoryFlagSource(); + tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withCurator(curator) + .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) + .withFlagSource(flagSource) + .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index 4f6642610dd..9665e0095e9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -5,7 +5,6 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -15,12 +14,6 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.rpc.RpcRequestHandlerProvider; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; -import com.yahoo.vespa.config.server.session.SessionPreparer; -import com.yahoo.vespa.config.server.session.SessionTest; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; import org.junit.Rule; @@ -28,7 +21,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.IOException; -import java.util.Collections; +import java.util.List; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; @@ -39,13 +32,9 @@ import static org.junit.Assert.assertTrue; */ public class InjectedGlobalComponentRegistryTest { - private Curator curator; - private Metrics metrics; - private SessionPreparer sessionPreparer; private ConfigserverConfig configserverConfig; private RpcServer rpcServer; private ConfigDefinitionRepo defRepo; - private PermanentApplicationPackage permanentApplicationPackage; private GlobalComponentRegistry globalComponentRegistry; private ModelFactoryRegistry modelFactoryRegistry; private Zone zone; @@ -55,45 +44,35 @@ public class InjectedGlobalComponentRegistryTest { @Before public void setupRegistry() throws IOException { - curator = new MockCurator(); - ConfigCurator configCurator = ConfigCurator.create(curator); - metrics = Metrics.createTestMetrics(); - modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); + modelFactoryRegistry = new ModelFactoryRegistry(List.of(new VespaModelFactory(new NullConfigModelRegistry()))); configserverConfig = new ConfigserverConfig( new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())); - sessionPreparer = new SessionTest.MockSessionPreparer(); HostRegistry hostRegistry = new HostRegistry(); rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), hostRegistry, new ConfigRequestHostLivenessTracker(), new FileServer(temporaryFolder.newFolder("filereferences")), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); defRepo = new StaticConfigDefinitionRepo(); - permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); - HostProvisionerProvider hostProvisionerProvider = HostProvisionerProvider.withProvisioner(new MockProvisioner()); zone = Zone.defaultZone(); globalComponentRegistry = - new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, sessionPreparer, - rpcServer, configserverConfig, defRepo, permanentApplicationPackage, - hostProvisionerProvider, zone, - new ConfigServerDB(configserverConfig), new InMemoryFlagSource(), - new MockSecretStore(), hostRegistry); + new InjectedGlobalComponentRegistry(modelFactoryRegistry, + rpcServer, + configserverConfig, + defRepo, + zone, + new ConfigServerDB(configserverConfig)); } @Test public void testThatAllComponentsAreSetup() { assertThat(globalComponentRegistry.getModelFactoryRegistry(), is(modelFactoryRegistry)); - assertThat(globalComponentRegistry.getSessionPreparer(), is(sessionPreparer)); - assertThat(globalComponentRegistry.getMetrics(), is(metrics)); - assertThat(globalComponentRegistry.getCurator(), is(curator)); assertThat(globalComponentRegistry.getConfigserverConfig(), is(configserverConfig)); assertThat(globalComponentRegistry.getReloadListener().hashCode(), is(rpcServer.hashCode())); assertThat(globalComponentRegistry.getTenantListener().hashCode(), is(rpcServer.hashCode())); assertThat(globalComponentRegistry.getStaticConfigDefinitionRepo(), is(defRepo)); - assertThat(globalComponentRegistry.getPermanentApplicationPackage(), is(permanentApplicationPackage)); - assertThat(globalComponentRegistry.getZone(), is (zone)); - assertTrue(globalComponentRegistry.getHostProvisioner().isPresent()); + assertThat(globalComponentRegistry.getZone(), is(zone)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index 0221cf9abfc..24f59ee8675 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -2,37 +2,18 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.concurrent.InThreadExecutorService; -import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.application.TenantApplicationsTest; -import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; -import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; -import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.MockTenantListener; import com.yahoo.vespa.config.server.tenant.TenantListener; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; import java.nio.file.Files; import java.time.Clock; import java.util.Collections; -import java.util.Optional; -import java.util.concurrent.ExecutorService; import static com.yahoo.yolean.Exceptions.uncheck; @@ -41,65 +22,33 @@ import static com.yahoo.yolean.Exceptions.uncheck; */ public class TestComponentRegistry implements GlobalComponentRegistry { - private final Curator curator; - private final ConfigCurator configCurator; - private final Metrics metrics; - private final SessionPreparer sessionPreparer; private final ConfigserverConfig configserverConfig; private final ConfigDefinitionRepo defRepo; private final ReloadListener reloadListener; private final TenantListener tenantListener; - private final PermanentApplicationPackage permanentApplicationPackage; - private final FileDistributionFactory fileDistributionFactory; private final ModelFactoryRegistry modelFactoryRegistry; - private final Optional<Provisioner> hostProvisioner; private final Zone zone; private final Clock clock; private final ConfigServerDB configServerDB; - private final StripedExecutor<TenantName> zkWatcherExecutor; - private final ExecutorService zkCacheExecutor; - private final SecretStore secretStore; - private final FlagSource flagSource; - - private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, - ModelFactoryRegistry modelFactoryRegistry, - PermanentApplicationPackage permanentApplicationPackage, - FileDistributionFactory fileDistributionFactory, + + private TestComponentRegistry(ModelFactoryRegistry modelFactoryRegistry, ConfigserverConfig configserverConfig, - SessionPreparer sessionPreparer, - Optional<Provisioner> hostProvisioner, ConfigDefinitionRepo defRepo, ReloadListener reloadListener, TenantListener tenantListener, Zone zone, - Clock clock, - SecretStore secretStore, - FlagSource flagSource) { - this.curator = curator; - this.configCurator = configCurator; - this.metrics = metrics; + Clock clock) { this.configserverConfig = configserverConfig; this.reloadListener = reloadListener; this.tenantListener = tenantListener; this.defRepo = defRepo; - this.permanentApplicationPackage = permanentApplicationPackage; - this.fileDistributionFactory = fileDistributionFactory; this.modelFactoryRegistry = modelFactoryRegistry; - this.hostProvisioner = hostProvisioner; - this.sessionPreparer = sessionPreparer; this.zone = zone; this.clock = clock; this.configServerDB = new ConfigServerDB(configserverConfig); - this.zkWatcherExecutor = new StripedExecutor<>(new InThreadExecutorService()); - this.zkCacheExecutor = new InThreadExecutorService(); - this.secretStore = secretStore; - this.flagSource = flagSource; } public static class Builder { - - private Curator curator = new MockCurator(); - private Metrics metrics = Metrics.createTestMetrics(); private ConfigserverConfig configserverConfig = new ConfigserverConfig( new ConfigserverConfig.Builder() .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) @@ -108,44 +57,20 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private ConfigDefinitionRepo defRepo = new StaticConfigDefinitionRepo(); private ReloadListener reloadListener = new TenantApplicationsTest.MockReloadListener(); private final MockTenantListener tenantListener = new MockTenantListener(); - private Optional<PermanentApplicationPackage> permanentApplicationPackage = Optional.empty(); - private final Optional<FileDistributionFactory> fileDistributionFactory = Optional.empty(); private ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); - private Optional<Provisioner> hostProvisioner = Optional.empty(); private Zone zone = Zone.defaultZone(); private Clock clock = Clock.systemUTC(); - private FlagSource flagSource = new InMemoryFlagSource(); public Builder configServerConfig(ConfigserverConfig configserverConfig) { this.configserverConfig = configserverConfig; return this; } - public Builder curator(Curator curator) { - this.curator = curator; - return this; - } - - public Builder metrics(Metrics metrics) { - this.metrics = metrics; - return this; - } - public Builder modelFactoryRegistry(ModelFactoryRegistry modelFactoryRegistry) { this.modelFactoryRegistry = modelFactoryRegistry; return this; } - public Builder permanentApplicationPackage(PermanentApplicationPackage permanentApplicationPackage) { - this.permanentApplicationPackage = Optional.ofNullable(permanentApplicationPackage); - return this; - } - - public Builder provisioner(Provisioner provisioner) { - this.hostProvisioner = Optional.ofNullable(provisioner); - return this; - } - public Builder zone(Zone zone) { this.zone = zone; return this; @@ -161,44 +86,23 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } - public Builder flagSource(FlagSource flagSource) { - this.flagSource = flagSource; - return this; - } - public Builder configDefinitionRepo(ConfigDefinitionRepo configDefinitionRepo) { this.defRepo = configDefinitionRepo; return this; } public TestComponentRegistry build() { - final PermanentApplicationPackage permApp = this.permanentApplicationPackage - .orElse(new PermanentApplicationPackage(configserverConfig)); - FileDistributionFactory fileDistributionProvider = this.fileDistributionFactory - .orElse(new MockFileDistributionFactory(configserverConfig)); - HostProvisionerProvider hostProvisionerProvider = hostProvisioner. - map(HostProvisionerProvider::withProvisioner).orElseGet(HostProvisionerProvider::empty); - SecretStore secretStore = new MockSecretStore(); - SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionProvider, - hostProvisionerProvider, permApp, - configserverConfig, defRepo, curator, - zone, flagSource, secretStore); - return new TestComponentRegistry(curator, ConfigCurator.create(curator), metrics, modelFactoryRegistry, - permApp, fileDistributionProvider, configserverConfig, - sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, - zone, clock, secretStore, flagSource); + return new TestComponentRegistry(modelFactoryRegistry, + configserverConfig, + defRepo, + reloadListener, + tenantListener, + zone, + clock); } } @Override - public Curator getCurator() { return curator; } - @Override - public ConfigCurator getConfigCurator() { return configCurator; } - @Override - public Metrics getMetrics() { return metrics; } - @Override - public SessionPreparer getSessionPreparer() { return sessionPreparer; } - @Override public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } @Override public TenantListener getTenantListener() { return tenantListener; } @@ -207,14 +111,8 @@ public class TestComponentRegistry implements GlobalComponentRegistry { @Override public ConfigDefinitionRepo getStaticConfigDefinitionRepo() { return defRepo; } @Override - public PermanentApplicationPackage getPermanentApplicationPackage() { return permanentApplicationPackage; } - @Override public ModelFactoryRegistry getModelFactoryRegistry() { return modelFactoryRegistry; } @Override - public Optional<Provisioner> getHostProvisioner() { - return hostProvisioner; - } - @Override public Zone getZone() { return zone; } @@ -223,24 +121,4 @@ public class TestComponentRegistry implements GlobalComponentRegistry { @Override public ConfigServerDB getConfigServerDB() { return configServerDB;} - @Override - public StripedExecutor<TenantName> getZkWatcherExecutor() { - return zkWatcherExecutor; - } - - @Override - public FlagSource getFlagSource() { return flagSource; } - - @Override - public ExecutorService getZkCacheExecutor() { - return zkCacheExecutor; - } - - @Override - public SecretStore getSecretStore() { - return secretStore; - } - - public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 4543d3b5673..732c9385608 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ConfigserverConfig; @@ -17,6 +17,7 @@ import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; @@ -55,6 +56,7 @@ public class TenantApplicationsTest { private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); private final MockReloadListener listener = new MockReloadListener(); + private Curator curator; private CuratorFramework curatorFramework; private TestComponentRegistry componentRegistry; private TenantApplications applications; @@ -64,23 +66,26 @@ public class TenantApplicationsTest { @Before public void setup() throws IOException { - Curator curator = new MockCurator(); + curator = new MockCurator(); curatorFramework = curator.framework(); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) + .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .build(); componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) - .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) - .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .modelFactoryRegistry(createRegistry()) .reloadListener(listener) .build(); HostRegistry hostRegistry = new HostRegistry(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, hostRegistry); + TenantRepository tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withCurator(curator) + .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenantName); - applications = TenantApplications.create(componentRegistry, hostRegistry, tenantName); + applications = TenantApplications.create(componentRegistry, hostRegistry, tenantName, curator); } @Test @@ -173,7 +178,7 @@ public class TenantApplicationsTest { @Test public void testListConfigs() throws IOException, SAXException { - applications = TenantApplications.create(componentRegistry, new HostRegistry(), TenantName.defaultName()); + applications = TenantApplications.create(componentRegistry, new HostRegistry(), TenantName.defaultName(), new MockCurator()); assertdefaultAppNotFound(); VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); @@ -208,7 +213,7 @@ public class TenantApplicationsTest { } private TenantApplications createZKAppRepo() { - return TenantApplications.create(componentRegistry, new HostRegistry(), tenantName); + return TenantApplications.create(componentRegistry, new HostRegistry(), tenantName, curator); } private static ApplicationId createApplicationId(String name) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 54d23652349..eb8f633f4ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -26,14 +26,16 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; @@ -263,7 +265,7 @@ public class DeployTester { private Provisioner provisioner; private ConfigserverConfig configserverConfig; private Zone zone; - private Curator curator; + private Curator curator = new MockCurator(); private Metrics metrics; private List<ModelFactory> modelFactories; private Orchestrator orchestrator; @@ -284,13 +286,18 @@ public class DeployTester { TestComponentRegistry.Builder testComponentRegistryBuilder = new TestComponentRegistry.Builder() .clock(clock) .configServerConfig(configserverConfig) - .curator(Optional.ofNullable(curator).orElseGet(MockCurator::new)) .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) - .metrics(Optional.ofNullable(metrics).orElseGet(Metrics::createTestMetrics)) .zone(zone); - if (configserverConfig.hostedVespa()) testComponentRegistryBuilder.provisioner(provisioner); - TenantRepository tenantRepository = new TenantRepository(testComponentRegistryBuilder.build(), new HostRegistry()); + TestTenantRepository.Builder builder = new TestTenantRepository.Builder() + .withComponentRegistry(testComponentRegistryBuilder.build()) + .withCurator(curator) + .withMetrics(Optional.ofNullable(metrics).orElse(Metrics.createTestMetrics())) + .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)); + + if (configserverConfig.hostedVespa()) builder.withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, true)); + + TenantRepository tenantRepository = builder.build(); tenantRepository.addTenant(tenantName); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 6c5e25d0503..3631922f844 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -11,9 +11,9 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -59,7 +59,7 @@ public class HttpGetConfigHandlerTest { .configDefinitionRepo(new TestConfigDefinitionRepo()) .configServerConfig(configserverConfig) .build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index b55b250fc1c..cdf89eda367 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -12,10 +12,10 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HttpListConfigsHandler.ListConfigsResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -64,7 +64,7 @@ public class HttpListConfigsHandlerTest { .configDefinitionRepo(new TestConfigDefinitionRepo()) .configServerConfig(configserverConfig) .build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 06bac6497a2..02129a1b5cd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -12,12 +12,12 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -63,7 +63,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { .build(); Clock clock = componentRegistry.getClock(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(tenantName1); tenantRepository.addTenant(tenantName2); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 293219eade3..0e0fd5babd5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -26,7 +26,7 @@ import com.yahoo.vespa.config.server.application.ClusterReindexing.Status; import com.yahoo.vespa.config.server.application.HttpProxy; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -37,6 +37,7 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -109,14 +110,17 @@ public class ApplicationHandlerTest { .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .provisioner(provisioner) .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) .configServerConfig(configserverConfig) .clock(clock) .build(); - tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); - tenantRepository.addTenant(mytenantName); provisioner = new MockProvisioner(); + tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, false)) + .build(); + tenantRepository.addTenant(mytenantName); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 6ef1a0b63cd..671faa42f7b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -14,11 +14,11 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -57,7 +57,7 @@ public class HostHandlerTest { .zone(zone) .configServerConfig(configserverConfig) .build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(mytenant); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 68cd474412f..d32734cd571 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -13,13 +13,14 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpConfigRequest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -67,11 +68,15 @@ public class HttpGetConfigHandlerTest { .configDefinitionRepo(new TestConfigDefinitionRepo()) .configServerConfig(configserverConfig) .build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + MockProvisioner provisioner = new MockProvisioner(); + TenantRepository tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, false)) + .build(); tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) + .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 985a88ceed2..cf04c8df56c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler.ListConfigsResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -71,7 +72,7 @@ public class HttpListConfigsHandlerTest { .configDefinitionRepo(new TestConfigDefinitionRepo()) .configServerConfig(configserverConfig) .build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index 232243c2b74..e3930c2784d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -12,9 +12,9 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Test; @@ -42,7 +42,7 @@ public class ListApplicationsHandlerTest { @Before public void setup() { - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(mytenant); tenantRepository.addTenant(foobar); applicationRepo = tenantRepository.getTenant(mytenant).getApplicationRepo(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 8e9ef42352c..24176d17c38 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.model.TestModelFactory; @@ -24,7 +23,7 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.model.VespaModelFactory; import org.hamcrest.core.Is; import org.junit.Before; @@ -76,11 +75,10 @@ public class SessionActiveHandlerTest { .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); componentRegistry = new TestComponentRegistry.Builder() - .curator(new MockCurator()) .modelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) .configServerConfig(configserverConfig) .build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(tenantName); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index ce086cbfb15..867dbdfe65a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -14,12 +14,12 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -61,7 +61,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { .configServerConfig(configserverConfig) .build(); - tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); tenantRepository.addTenant(tenantName); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 7efd448c3ed..56afe1e4a08 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -11,11 +11,11 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -63,7 +63,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Before public void setupRepo() { - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new MockProvisioner()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index ed65d8294b8..b87f24cf06d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -19,11 +19,11 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; @@ -77,12 +77,14 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) .configServerConfig(configserverConfig) .build(); Clock clock = componentRegistry.getClock(); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); - tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withCurator(curator) + .build(); tenantRepository.addTenant(tenant); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index a8cc989c802..d9f5f094023 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -1,40 +1,41 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; - import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.NotFoundException; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.vespa.config.server.http.BadRequestException; -import com.yahoo.vespa.config.server.http.NotFoundException; import org.junit.rules.TemporaryFolder; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + public class TenantHandlerTest { private static final File testApp = new File("src/test/apps/app"); @@ -54,12 +55,10 @@ public class TenantHandlerTest { .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() - .curator(new MockCurator()) - .configServerConfig(configserverConfig) - .build(), - new HostRegistry()); - + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new MockProvisioner()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 1d8f60bb683..3e9cbc042bd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -11,10 +11,11 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.rules.TemporaryFolder; @@ -41,13 +42,14 @@ class MaintainerTester { .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) .clock(clock) .configServerConfig(configserverConfig) - .provisioner(provisioner) .modelFactoryRegistry(new ModelFactoryRegistry(List.of(new DeployTester.CountingModelFactory(clock)))) .build(); - tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, true)) + .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(provisioner) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 5a41eff3cc9..55a6572ce3b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -25,10 +25,8 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index 06c224b5a09..ffc60d01732 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.After; import org.junit.rules.TemporaryFolder; @@ -90,7 +91,10 @@ public class RpcTester implements AutoCloseable { .configServerConfig(configserverConfig) .reloadListener(rpcServer) .build(); - tenantRepository = new TenantRepository(componentRegistry, hostRegistry); + tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(componentRegistry) + .withHostRegistry(hostRegistry) + .build(); tenantRepository.addTenant(tenantName); startRpcServer(); applicationRepository = new ApplicationRepository.Builder() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 90d3bddc88d..5a49090031c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudgetTest; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; +import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.model.TestModelFactory; @@ -100,7 +101,6 @@ public class SessionPreparerTest { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) .configServerConfig(new ConfigserverConfig.Builder() .fileReferencesDir(folder.newFolder().getAbsolutePath()) .configServerDBDir(folder.newFolder().getAbsolutePath()) @@ -124,7 +124,7 @@ public class SessionPreparerTest { HostProvisionerProvider hostProvisionerProvider) { return new SessionPreparer( modelFactoryRegistry, - componentRegistry.getFileDistributionFactory(), + new MockFileDistributionFactory(componentRegistry.getConfigserverConfig()), hostProvisionerProvider, new PermanentApplicationPackage(componentRegistry.getConfigserverConfig()), componentRegistry.getConfigserverConfig(), @@ -289,27 +289,6 @@ public class SessionPreparerTest { } @Test - public void require_that_tlssecretkey_is_written() throws IOException { - var tlskey = "vespa.tlskeys.tenant1--app1"; - var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - - secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); - secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); - - prepare(new File("src/test/resources/deploy/hosted-app"), params); - - // Read from zk and verify cert and key are available - Path tenantPath = TenantRepository.getTenantPath(applicationId.tenant()); - Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) - .readEndpointCertificateMetadata(applicationId) - .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test public void require_that_endpoint_certificate_metadata_is_written() throws IOException { var applicationId = applicationId("test"); var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); @@ -329,25 +308,24 @@ public class SessionPreparerTest { } @Test(expected = CertificateNotReadyException.class) - public void require_that_tlssecretkey_is_missing_when_not_in_secretstore() throws IOException { - var tlskey = "vespa.tlskeys.tenant1--app1"; + public void endpoint_certificate_is_missing_when_not_in_secretstore() throws IOException { var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); } @Test(expected = CertificateNotReadyException.class) - public void require_that_tlssecretkey_is_missing_when_certificate_not_in_secretstore() throws IOException { + public void endpoint_certificate_is_missing_when_certificate_not_in_secretstore() throws IOException { var tlskey = "vespa.tlskeys.tenant1--app1"; var applicationId = applicationId("test"); - var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - secretStore.put(tlskey+"-key", "KEY"); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); + secretStore.put(tlskey+"-key", 7, "KEY"); prepare(new File("src/test/resources/deploy/hosted-app"), params); } @Test(expected = LoadBalancerServiceException.class) public void require_that_conflict_is_returned_when_creating_load_balancer_fails() throws IOException { - preparer = createPreparer(HostProvisionerProvider.withProvisioner(new MockProvisioner().transientFailureOnPrepare())); + preparer = createPreparer(HostProvisionerProvider.withProvisioner(new MockProvisioner().transientFailureOnPrepare(), true)); var params = new PrepareParams.Builder().applicationId(applicationId("test")).build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index b2cfd7015ba..0ca04d51107 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -20,10 +20,11 @@ import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; @@ -82,22 +83,26 @@ public class SessionRepositoryTest { private void setup(FlagSource flagSource, TestComponentRegistry.Builder componentRegistryBuilder) throws Exception { curator = new MockCurator(); File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(configserverDbDir.getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .sessionLifetime(5) + .build(); GlobalComponentRegistry globalComponentRegistry = componentRegistryBuilder - .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(configserverDbDir.getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) - .sessionLifetime(5) - .build()) - .flagSource(flagSource) + .configServerConfig(configserverConfig) + .build(); + tenantRepository = new TestTenantRepository.Builder() + .withComponentRegistry(globalComponentRegistry) + .withCurator(curator) + .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) .build(); - tenantRepository = new TenantRepository(globalComponentRegistry, new HostRegistry()); tenantRepository.addTenant(SessionRepositoryTest.tenantName); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withFlagSource(flagSource) .build(); sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java index 829e47bdb42..1a4c7c9669e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java @@ -53,18 +53,6 @@ public class EndpointCertificateMetadataStoreTest { } @Test - public void reads_string_format() { - curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes()); - - // Read from zk and verify cert and key are available - var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) - .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); - assertTrue(endpointCertificateSecrets.isPresent()); - assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); - assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); - } - - @Test public void reads_object_format() { curator.set(endpointCertificateMetadataPath, "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index dbc352128f5..c843b5f6119 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -1,8 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.concurrent.InThreadExecutorService; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -13,16 +15,22 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.MockProvisioner; +import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.application.TenantApplicationsTest; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.After; import org.junit.Before; @@ -49,7 +57,6 @@ public class TenantRepositoryTest { private static final TenantName tenant3 = TenantName.from("tenant3"); private TenantRepository tenantRepository; - private TestComponentRegistry globalComponentRegistry; private TenantApplicationsTest.MockReloadListener listener; private MockTenantListener tenantListener; private Curator curator; @@ -63,11 +70,13 @@ public class TenantRepositoryTest { @Before public void setupSessions() { curator = new MockCurator(); - globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - listener = (TenantApplicationsTest.MockReloadListener)globalComponentRegistry.getReloadListener(); - tenantListener = (MockTenantListener)globalComponentRegistry.getTenantListener(); + TestComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder().build(); + listener = (TenantApplicationsTest.MockReloadListener) globalComponentRegistry.getReloadListener(); + tenantListener = (MockTenantListener) globalComponentRegistry.getTenantListener(); assertFalse(tenantListener.tenantsLoaded); - tenantRepository = new TenantRepository(globalComponentRegistry, new HostRegistry()); + tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(globalComponentRegistry) + .withCurator(curator) + .build(); assertTrue(tenantListener.tenantsLoaded); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); @@ -185,13 +194,11 @@ public class TenantRepositoryTest { } private void assertZooKeeperTenantPathExists(TenantName tenantName) throws Exception { - assertNotNull(globalComponentRegistry.getCurator().framework() - .checkExists().forPath(TenantRepository.getTenantPath(tenantName).getAbsolute())); + assertNotNull(curator.framework().checkExists().forPath(TenantRepository.getTenantPath(tenantName).getAbsolute())); } private GlobalComponentRegistry createComponentRegistry() throws IOException { return new TestComponentRegistry.Builder() - .curator(new MockCurator()) .configServerConfig(new ConfigserverConfig(new ConfigserverConfig.Builder() .configDefinitionsDir(temporaryFolder.newFolder("configdefs").getAbsolutePath()) .configServerDBDir(temporaryFolder.newFolder("configserverdb").getAbsolutePath()))) @@ -201,8 +208,17 @@ public class TenantRepositoryTest { private static class FailingDuringBootstrapTenantRepository extends TenantRepository { - public FailingDuringBootstrapTenantRepository(GlobalComponentRegistry globalComponentRegistry) { - super(globalComponentRegistry, new HostRegistry()); + public FailingDuringBootstrapTenantRepository(GlobalComponentRegistry componentRegistry) { + super(componentRegistry, + new HostRegistry(), + new MockCurator(), + Metrics.createTestMetrics(), + new StripedExecutor<>(new InThreadExecutorService()), + new FileDistributionFactory(new ConfigserverConfig.Builder().build()), + new InMemoryFlagSource(), + new InThreadExecutorService(), + new MockSecretStore(), + HostProvisionerProvider.withProvisioner(new MockProvisioner(), false)); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java index 1b754ca6695..75d1b848f50 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java @@ -33,7 +33,7 @@ public class TenantTest { } private Tenant createTenant(String name) { - TenantRepository tenantRepository = new TenantRepository(componentRegistry, new HostRegistry()); + TenantRepository tenantRepository = new TestTenantRepository.Builder().withComponentRegistry(componentRegistry).build(); TenantName tenantName = TenantName.from(name); tenantRepository.addTenant(tenantName); return tenantRepository.getTenant(tenantName); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java new file mode 100644 index 00000000000..b4922e1f163 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java @@ -0,0 +1,101 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.concurrent.InThreadExecutorService; +import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.MockSecretStore; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; +import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; + +/** + * + * @author hmusum + */ +public class TestTenantRepository extends TenantRepository { + + public TestTenantRepository(GlobalComponentRegistry componentRegistry, + HostRegistry hostRegistry, + Curator curator, + Metrics metrics, + FileDistributionFactory fileDistributionFactory, + FlagSource flagSource, + HostProvisionerProvider hostProvisionerProvider) { + super(componentRegistry, + hostRegistry, + curator, + metrics, + new StripedExecutor<>(new InThreadExecutorService()), + fileDistributionFactory, + flagSource, + new InThreadExecutorService(), + new MockSecretStore(), + hostProvisionerProvider); + } + + public static class Builder { + + GlobalComponentRegistry componentRegistry; + HostRegistry hostRegistry = new HostRegistry(); + Curator curator = new MockCurator(); + Metrics metrics = Metrics.createTestMetrics(); + FileDistributionFactory fileDistributionFactory = null; + FlagSource flagSource = new InMemoryFlagSource(); + HostProvisionerProvider hostProvisionerProvider = HostProvisionerProvider.empty(); + + public Builder withFlagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + + public Builder withComponentRegistry(GlobalComponentRegistry componentRegistry) { + this.componentRegistry = componentRegistry; + return this; + } + + public Builder withHostRegistry(HostRegistry hostRegistry) { + this.hostRegistry = hostRegistry; + return this; + } + + public Builder withCurator(Curator curator) { + this.curator = curator; + return this; + } + + public Builder withMetrics(Metrics metrics) { + this.metrics = metrics; + return this; + } + + public Builder withFileDistributionFactory(FileDistributionFactory fileDistributionFactory) { + this.fileDistributionFactory = fileDistributionFactory; + return this; + } + + public Builder withHostProvisionerProvider(HostProvisionerProvider hostProvisionerProvider) { + this.hostProvisionerProvider = hostProvisionerProvider; + return this; + } + + public TenantRepository build() { + if (fileDistributionFactory == null) + fileDistributionFactory = new FileDistributionFactory(componentRegistry.getConfigserverConfig()); + return new TestTenantRepository(componentRegistry, + hostRegistry, + curator, + metrics, + fileDistributionFactory, + flagSource, + hostProvisionerProvider); + } + + } + +} diff --git a/container-core-config/src/main/resources/configdefinitions/container.core.access-log.def b/container-core-config/src/main/resources/configdefinitions/container.core.access-log.def index 08ea6ec4884..0c0956f2504 100644 --- a/container-core-config/src/main/resources/configdefinitions/container.core.access-log.def +++ b/container-core-config/src/main/resources/configdefinitions/container.core.access-log.def @@ -15,3 +15,6 @@ fileHandler.symlink string default="" # compress the previous access log after rotation fileHandler.compressOnRotation bool default=true + +# Compression format +fileHandler.compressionFormat enum {GZIP, ZSTD} default=GZIP diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index e610e5505af..4d2cafa3e48 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -17,16 +17,13 @@ public class EndpointCertificateMetadata { private final String certName; private final int version; private final long lastRequested; - // TODO: make these fields required once all certs have them stored - private final Optional<String> request_id; - private final Optional<List<String>> requestedDnsSans; - private final Optional<String> issuer; + private final String request_id; + private final List<String> requestedDnsSans; + private final String issuer; + private final Optional<Long> expiry; + private final Optional<Long> lastRefreshed; - public EndpointCertificateMetadata(String keyName, String certName, int version, long lastRequested) { - this(keyName, certName, version, lastRequested, Optional.empty(), Optional.empty(), Optional.empty()); - } - - public EndpointCertificateMetadata(String keyName, String certName, int version, long lastRequested, Optional<String> request_id, Optional<List<String>> requestedDnsSans, Optional<String> issuer) { + public EndpointCertificateMetadata(String keyName, String certName, int version, long lastRequested, String request_id, List<String> requestedDnsSans, String issuer, Optional<Long> expiry, Optional<Long> lastRefreshed) { this.keyName = keyName; this.certName = certName; this.version = version; @@ -34,6 +31,8 @@ public class EndpointCertificateMetadata { this.request_id = request_id; this.requestedDnsSans = requestedDnsSans; this.issuer = issuer; + this.expiry = expiry; + this.lastRefreshed = lastRefreshed; } public String keyName() { @@ -52,18 +51,26 @@ public class EndpointCertificateMetadata { return lastRequested; } - public Optional<String> request_id() { + public String request_id() { return request_id; } - public Optional<List<String>> requestedDnsSans() { + public List<String> requestedDnsSans() { return requestedDnsSans; } - public Optional<String> issuer() { + public String issuer() { return issuer; } + public Optional<Long> expiry() { + return expiry; + } + + public Optional<Long> lastRefreshed() { + return lastRefreshed; + } + public EndpointCertificateMetadata withVersion(int version) { return new EndpointCertificateMetadata( this.keyName, @@ -72,8 +79,9 @@ public class EndpointCertificateMetadata { this.lastRequested, this.request_id, this.requestedDnsSans, - this.issuer - ); + this.issuer, + this.expiry, + this.lastRefreshed); } public EndpointCertificateMetadata withLastRequested(long lastRequested) { @@ -84,8 +92,22 @@ public class EndpointCertificateMetadata { lastRequested, this.request_id, this.requestedDnsSans, - this.issuer - ); + this.issuer, + this.expiry, + this.lastRefreshed); + } + + public EndpointCertificateMetadata withLastRefreshed(long lastRefreshed) { + return new EndpointCertificateMetadata( + this.keyName, + this.certName, + this.version, + this.lastRequested, + this.request_id, + this.requestedDnsSans, + this.issuer, + this.expiry, + Optional.of(lastRefreshed)); } @Override @@ -98,6 +120,8 @@ public class EndpointCertificateMetadata { ", request_id=" + request_id + ", requestedDnsSans=" + requestedDnsSans + ", issuer=" + issuer + + ", expiry=" + expiry + + ", lastRefreshed=" + lastRefreshed + '}'; } @@ -112,11 +136,13 @@ public class EndpointCertificateMetadata { certName.equals(that.certName) && request_id.equals(that.request_id) && requestedDnsSans.equals(that.requestedDnsSans) && - issuer.equals(that.issuer); + issuer.equals(that.issuer) && + expiry.equals(that.expiry) && + lastRefreshed.equals(that.lastRefreshed); } @Override public int hashCode() { - return Objects.hash(keyName, certName, version, lastRequested, request_id, requestedDnsSans, issuer); + return Objects.hash(keyName, certName, version, lastRequested, request_id, requestedDnsSans, issuer, expiry, lastRefreshed); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java index 8c63613ec91..b5ee78251f0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.certificates; import com.yahoo.config.provision.ApplicationId; +import java.time.Instant; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -25,7 +26,10 @@ public class EndpointCertificateMock implements EndpointCertificateProvider { this.dnsNames.put(applicationId, dnsNames); String endpointCertificatePrefix = String.format("vespa.tls.%s.%s.%s", applicationId.tenant(), applicationId.application(), applicationId.instance()); - return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0, 0, Optional.of("mock-id-string"), Optional.of(dnsNames), Optional.of("mockCa")); + long epochSecond = Instant.now().getEpochSecond(); + long inAnHour = epochSecond + 3600; + return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0, 0, + "mock-id-string", dnsNames, "mockCa", Optional.of(inAnHour), Optional.of(epochSecond)); } @Override diff --git a/controller-server/pom.xml b/controller-server/pom.xml index ea3bbcf1e49..906cc5c0ee1 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -145,6 +145,17 @@ </dependency> <dependency> + <groupId>com.auth0</groupId> + <artifactId>java-jwt</artifactId> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-model-api</artifactId> <version>${project.version}</version> @@ -203,6 +214,13 @@ <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <build> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 81cba77b3c3..aa5f0ae0fdc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -79,6 +79,7 @@ public class Controller extends AbstractComponent { private final Metric metric; private final RoutingController routingController; private final ControllerConfig controllerConfig; + private final SecretStore secretStore; /** * Creates a controller @@ -115,6 +116,7 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); this.controllerConfig = controllerConfig; + this.secretStore = secretStore; // Record the version of this controller curator().writeControllerVersion(this.hostname(), ControllerVersion.CURRENT); @@ -281,6 +283,10 @@ public class Controller extends AbstractComponent { return metric; } + public SecretStore secretStore() { + return secretStore; + } + private Set<CloudName> clouds() { return zoneRegistry.zones().all().zones().stream() .map(ZoneApi::getCloudName) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index b998ed29b71..e45bda0708e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -13,11 +13,13 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import java.security.Principal; import java.security.PublicKey; +import java.time.Instant; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -31,9 +33,13 @@ import static java.util.Objects.requireNonNull; public abstract class LockedTenant { final TenantName name; + final Instant createdAt; + final LastLoginInfo lastLoginInfo; - private LockedTenant(TenantName name) { + private LockedTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo) { this.name = requireNonNull(name); + this.createdAt = requireNonNull(createdAt); + this.lastLoginInfo = requireNonNull(lastLoginInfo); } static LockedTenant of(Tenant tenant, Lock lock) { @@ -47,6 +53,8 @@ public abstract class LockedTenant { /** Returns a read-only copy of this */ public abstract Tenant get(); + public abstract LockedTenant with(LastLoginInfo lastLoginInfo); + @Override public String toString() { return "tenant '" + name + "'"; @@ -61,8 +69,9 @@ public abstract class LockedTenant { private final Optional<PropertyId> propertyId; private final Optional<Contact> contact; - private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, Optional<Contact> contact) { - super(name); + private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, + Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo) { + super(name, createdAt, lastLoginInfo); this.domain = domain; this.property = property; this.propertyId = propertyId; @@ -70,28 +79,33 @@ public abstract class LockedTenant { } private Athenz(AthenzTenant tenant) { - this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact()); + this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt(), tenant.lastLoginInfo()); } @Override public AthenzTenant get() { - return new AthenzTenant(name, domain, property, propertyId, contact); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } public Athenz with(AthenzDomain domain) { - return new Athenz(name, domain, property, propertyId, contact); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } public Athenz with(Property property) { - return new Athenz(name, domain, property, propertyId, contact); + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } public Athenz with(PropertyId propertyId) { - return new Athenz(name, domain, property, Optional.of(propertyId), contact); + return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt, lastLoginInfo); } public Athenz with(Contact contact) { - return new Athenz(name, domain, property, propertyId, Optional.of(contact)); + return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt, lastLoginInfo); + } + + @Override + public LockedTenant with(LastLoginInfo lastLoginInfo) { + return new Athenz(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } } @@ -104,20 +118,20 @@ public abstract class LockedTenant { private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; - private Cloud(TenantName name, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { - super(name); + private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { + super(name, createdAt, lastLoginInfo); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; } private Cloud(CloudTenant tenant) { - this(tenant.name(), Optional.empty(), tenant.developerKeys(), tenant.info()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Optional.empty(), tenant.developerKeys(), tenant.info()); } @Override public CloudTenant get() { - return new CloudTenant(name, creator, developerKeys, info); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -125,17 +139,22 @@ public abstract class LockedTenant { if (keys.containsKey(key)) throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); keys.put(key, principal); - return new Cloud(name, creator, keys, info); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, creator, keys, info); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, creator, developerKeys, newInfo); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo); + } + + @Override + public LockedTenant with(LastLoginInfo lastLoginInfo) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 5560796a97d..4c9cf4f105f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; @@ -99,7 +100,7 @@ public class TenantController { try (Lock lock = lock(tenantSpec.tenant())) { requireNonExistent(tenantSpec.tenant()); TenantId.validate(tenantSpec.tenant().value()); - curator.writeTenant(accessControl.createTenant(tenantSpec, credentials, asList())); + curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); } } @@ -121,6 +122,22 @@ public class TenantController { } } + /** + * Update last login times for the given tenant at the given user levers with the given instant, but only if the + * new instant is later + */ + public void updateLastLogin(TenantName tenantName, List<LastLoginInfo.UserLevel> userLevels, Instant loggedInAt) { + try (Lock lock = lock(tenantName)) { + Tenant tenant = require(tenantName); + LastLoginInfo loginInfo = tenant.lastLoginInfo(); + for (LastLoginInfo.UserLevel userLevel : userLevels) + loginInfo = loginInfo.withLastLoginIfLater(userLevel, loggedInAt); + + if (tenant.lastLoginInfo().equals(loginInfo)) return; // no change + curator.writeTenant(LockedTenant.of(tenant, lock).with(loginInfo).get()); + } + } + /** Deletes the given tenant. */ public void delete(TenantName tenant, Credentials credentials) { try (Lock lock = lock(tenant)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index c8cce94d479..3c035ef01ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -6,7 +6,6 @@ import com.google.common.cache.CacheLoader; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -33,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.ForbiddenException; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -41,6 +41,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Predicate; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -79,7 +80,7 @@ public class AthenzFacade implements AccessControl { } @Override - public Tenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { + public Tenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing) { AthenzTenantSpec spec = (AthenzTenantSpec) tenantSpec; AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; AthenzDomain domain = spec.domain(); @@ -94,7 +95,8 @@ public class AthenzFacade implements AccessControl { AthenzTenant tenant = AthenzTenant.create(spec.tenant(), domain, spec.property(), - spec.propertyId()); + spec.propertyId(), + createdAt); if (existingWithSameDomain.isPresent()) { // Throw if domain is already taken. throw new IllegalArgumentException("Could not create tenant '" + spec.tenant().value() + @@ -123,11 +125,16 @@ public class AthenzFacade implements AccessControl { .filter(tenant -> tenant.type() == Tenant.Type.athenz && newDomain.equals(((AthenzTenant) tenant).domain())) .findAny(); + Instant createdAt = existing.stream() + .filter(tenant -> tenant.name().equals(spec.tenant())) + .findAny().orElseThrow() // Should not happen, we assert that the tenant exists before the method is called + .createdAt(); Tenant tenant = AthenzTenant.create(spec.tenant(), newDomain, spec.property(), - spec.propertyId()); + spec.propertyId(), + createdAt); if (existingWithSameDomain.isPresent()) { // Throw if domain taken by someone else, or do nothing if taken by this tenant. if ( ! existingWithSameDomain.get().equals(tenant)) // Equality by name. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index ac146858145..76fa52c0706 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -14,6 +14,7 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -67,6 +68,7 @@ public class EndpointCertificateManager { private final BooleanFlag validateEndpointCertificates; private final StringFlag deleteUnusedEndpointCertificates; private final BooleanFlag endpointCertInSharedRouting; + private final BooleanFlag useEndpointCertificateMaintainer; public EndpointCertificateManager(ZoneRegistry zoneRegistry, CuratorDb curator, @@ -81,6 +83,7 @@ public class EndpointCertificateManager { this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource); this.deleteUnusedEndpointCertificates = Flags.DELETE_UNUSED_ENDPOINT_CERTIFICATES.bindTo(flagSource); this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); + this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(flagSource); Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { try { this.deleteUnusedCertificates(); @@ -117,10 +120,9 @@ public class EndpointCertificateManager { return Optional.of(provisionedCertificateMetadata); } - // Reprovision certificate if it is missing SANs for the zone we are deploying to - var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans(); + // Re-provision certificate if it is missing SANs for the zone we are deploying to var requiredSansForZone = dnsNamesOf(instance.id(), zone); - if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) { + if (!currentCertificateMetadata.get().requestedDnsSans().containsAll(requiredSansForZone)) { var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec); curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata); // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry @@ -128,13 +130,17 @@ public class EndpointCertificateManager { return Optional.of(reprovisionedCertificateMetadata); } - // Look for and use refreshed certificate - var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); - if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { - var refreshedCertificateMetadata = currentCertificateMetadata.get().withVersion(latestAvailableVersion.getAsInt()); - validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); - curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); - return Optional.of(refreshedCertificateMetadata); + if (!useEndpointCertificateMaintainer.value()) { + // Look for and use refreshed certificate + var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { + var refreshedCertificateMetadata = currentCertificateMetadata.get() + .withVersion(latestAvailableVersion.getAsInt()) + .withLastRefreshed(clock.instant().getEpochSecond()); + validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); + curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); + return Optional.of(refreshedCertificateMetadata); + } } validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone); @@ -149,23 +155,31 @@ public class EndpointCertificateManager { private void deleteUnusedCertificates() { CleanupMode mode = CleanupMode.valueOf(deleteUnusedEndpointCertificates.value().toUpperCase()); - if (mode == CleanupMode.DISABLE) return; + if (mode == CleanupMode.DISABLE || useEndpointCertificateMaintainer.value()) return; var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS); curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() - + " has not been requested in a month and app has no deployments" - + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : "")); - if (mode == CleanupMode.ENABLE) { - endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); - curator.deleteEndpointCertificateMetadata(applicationId); + try (Lock lock = lock(applicationId)) { + if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { + log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + + " has not been requested in a month and app has no deployments" + + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : "")); + if (mode == CleanupMode.ENABLE) { + endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); + curator.deleteEndpointCertificateMetadata(applicationId); + } + } } } }); } + private Lock lock(ApplicationId applicationId) { + return curator.lock(TenantAndApplicationId.from(applicationId)); + } + private boolean hasNoDeployments(ApplicationId applicationId) { var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId)) .flatMap(app -> app.get(applicationId.instance())) @@ -187,8 +201,7 @@ public class EndpointCertificateManager { private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) { List<String> currentlyPresentNames = currentMetadata.isPresent() ? - currentMetadata.get().requestedDnsSans().orElseThrow(() -> new RuntimeException("Certificate metadata exists but SANs are not present!")) - : Collections.emptyList(); + currentMetadata.get().requestedDnsSans() : Collections.emptyList(); var requiredZones = new LinkedHashSet<>(Set.of(deploymentZone)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index ff2a1963967..56b7e5b2e46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -63,6 +63,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ContainerImageExpirer(controller, intervals.containerImageExpirer)); maintainers.add(new HostSwitchUpdater(controller, intervals.hostSwitchUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); + maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); } public Upgrader upgrader() { return upgrader; } @@ -109,6 +110,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration containerImageExpirer; private final Duration hostSwitchUpdater; private final Duration reindexingTriggerer; + private final Duration endpointCertificateMaintainer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -132,6 +134,7 @@ public class ControllerMaintenance extends AbstractComponent { this.containerImageExpirer = duration(2, HOURS); this.hostSwitchUpdater = duration(12, HOURS); this.reindexingTriggerer = duration(1, HOURS); + this.endpointCertificateMaintainer = duration(12, HOURS); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java new file mode 100644 index 00000000000..cbb5363aff9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -0,0 +1,157 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.google.common.collect.Sets; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.Predicate; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates + * + * @author andreer + */ +public class EndpointCertificateMaintainer extends ControllerMaintainer { + + private static final Logger log = Logger.getLogger(EndpointCertificateMaintainer.class.getName()); + + private final DeploymentTrigger deploymentTrigger; + private final Clock clock; + private final CuratorDb curator; + private final SecretStore secretStore; + private final EndpointCertificateProvider endpointCertificateProvider; + private final BooleanFlag useEndpointCertificateMaintainer; + + public EndpointCertificateMaintainer(Controller controller, Duration interval) { + super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); + this.deploymentTrigger = controller.applications().deploymentTrigger(); + this.clock = controller.clock(); + this.secretStore = controller.secretStore(); + this.curator = controller().curator(); + this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); + this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(controller().flagSource()); + } + + @Override + protected boolean maintain() { + + if (!useEndpointCertificateMaintainer.value()) + return true; // handled by EndpointCertificateManager for now + + try { + // In order of importance + deployRefreshedCertificates(); + updateRefreshedCertificates(); + deleteUnusedCertificates(); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); + return false; + } + + return true; + } + + private void updateRefreshedCertificates() { + curator.readAllEndpointCertificateMetadata().forEach(((applicationId, endpointCertificateMetadata) -> { + // Look for and use refreshed certificate + var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata); + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) { + var refreshedCertificateMetadata = endpointCertificateMetadata + .withVersion(latestAvailableVersion.getAsInt()) + .withLastRefreshed(clock.instant().getEpochSecond()); + try (Lock lock = lock(applicationId)) { + if (Optional.of(endpointCertificateMetadata).equals(curator.readEndpointCertificateMetadata(applicationId))) { + curator.writeEndpointCertificateMetadata(applicationId, refreshedCertificateMetadata); // Certificate not validated here, but on deploy. + } + } + } + })); + } + + /** + * If it's been a week since the cert has been refreshed, re-trigger all prod deployment jobs. + */ + private void deployRefreshedCertificates() { + var now = clock.instant(); + curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) -> + endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> { + Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); + if (now.isAfter(refreshTime.plus(7, ChronoUnit.DAYS))) { + + controller().jobController().jobs(applicationId).forEach(job -> + controller().jobController().jobStatus(new JobId(applicationId, JobType.fromJobName(job.jobName()))).lastTriggered().ifPresent(run -> { + if (run.start().isBefore(refreshTime) && job.isProduction() && job.isDeployment()) { + deploymentTrigger.reTrigger(applicationId, job); + log.info("Re-triggering deployment job " + job.jobName() + " for instance " + + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); + } + })); + } + })); + } + + private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) { + try { + var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName())); + var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName())); + return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); + } catch (SecretNotFoundException s) { + return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version + } + } + + private void deleteUnusedCertificates() { + var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS); + curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { + var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); + if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { + try (Lock lock = lock(applicationId)) { + if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { + log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + + " has not been requested in a month and app has no deployments, deleting from provider and ZK"); + endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); + curator.deleteEndpointCertificateMetadata(applicationId); + } + } + } + }); + } + + private Lock lock(ApplicationId applicationId) { + return curator.lock(TenantAndApplicationId.from(applicationId)); + } + + private boolean hasNoDeployments(ApplicationId applicationId) { + var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId)) + .flatMap(app -> app.get(applicationId.instance())) + .map(Instance::deployments); + + return deployments.isEmpty() || deployments.get().size() == 0; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 805803c13aa..e2d1c5bf7c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -52,6 +53,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -296,7 +298,10 @@ public class CuratorDb { } public Optional<Tenant> readTenant(TenantName name) { - return readSlime(tenantPath(name)).map(tenantSerializer::tenantFrom); + Supplier<Instant> tenantCreateTimeSupplier = () -> curator.getStat(tenantPath(name)) + .map(stat -> Instant.ofEpochMilli(stat.getCtime())) + .orElse(Instant.parse("2021-01-01T00:00:00Z")); + return readSlime(tenantPath(name)).map(bytes -> tenantSerializer.tenantFrom(bytes, tenantCreateTimeSupplier)); } public List<Tenant> readTenants() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index ba882ef7985..19f9542c679 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -7,7 +7,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; -import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -16,7 +15,7 @@ import java.util.stream.IntStream; * (de)serializes endpoint certificate metadata * <p> * A copy of package com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadata, - * but will soon be extended as we need to store some more information in the controller. + * but with additional fields as we need to store some more information in the controller. * * @author andreer */ @@ -36,6 +35,8 @@ public class EndpointCertificateMetadataSerializer { private final static String requestIdField = "requestId"; private final static String requestedDnsSansField = "requestedDnsSans"; private final static String issuerField = "issuer"; + private final static String expiryField = "expiry"; + private final static String lastRefreshedField = "lastRefreshed"; public static Slime toSlime(EndpointCertificateMetadata metadata) { Slime slime = new Slime(); @@ -44,13 +45,12 @@ public class EndpointCertificateMetadataSerializer { object.setString(certNameField, metadata.certName()); object.setLong(versionField, metadata.version()); object.setLong(lastRequestedField, metadata.lastRequested()); - - metadata.request_id().ifPresent(id -> object.setString(requestIdField, id)); - metadata.requestedDnsSans().ifPresent(sans -> { - Cursor cursor = object.setArray(requestedDnsSansField); - sans.forEach(cursor::addString); - }); - metadata.issuer().ifPresent(id -> object.setString(issuerField, id)); + object.setString(requestIdField, metadata.request_id()); + var cursor = object.setArray(requestedDnsSansField); + metadata.requestedDnsSans().forEach(cursor::addString); + object.setString(issuerField, metadata.issuer()); + metadata.expiry().ifPresent(expiry -> object.setLong(expiryField, expiry)); + metadata.lastRefreshed().ifPresent(refreshTime -> object.setLong(lastRefreshedField, refreshTime)); return slime; } @@ -58,32 +58,22 @@ public class EndpointCertificateMetadataSerializer { public static EndpointCertificateMetadata fromSlime(Inspector inspector) { if (inspector.type() != Type.OBJECT) throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); - Optional<String> request_id = inspector.field(requestIdField).valid() ? - Optional.of(inspector.field(requestIdField).asString()) : - Optional.empty(); - - Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? - Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) - .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : - Optional.empty(); - - Optional<String> issuer = inspector.field(issuerField).valid() ? - Optional.of(inspector.field(issuerField).asString()) : - Optional.empty(); - - long lastRequested = inspector.field(lastRequestedField).valid() ? - inspector.field(lastRequestedField).asLong() : - 1597200000L; // Wed Aug 12 02:40:00 UTC 2020 - // Not originally stored, so we default to when field was added return new EndpointCertificateMetadata( inspector.field(keyNameField).asString(), inspector.field(certNameField).asString(), Math.toIntExact(inspector.field(versionField).asLong()), - lastRequested, - request_id, - requestedDnsSans, - issuer); + inspector.field(lastRequestedField).asLong(), + inspector.field(requestIdField).asString(), + IntStream.range(0, inspector.field(requestedDnsSansField).entries()) + .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList()), + inspector.field(issuerField).asString(), + inspector.field(expiryField).valid() ? + Optional.of(inspector.field(expiryField).asLong()) : + Optional.empty(), + inspector.field(lastRefreshedField).valid() ? + Optional.of(inspector.field(lastRefreshedField).asLong()) : + Optional.empty()); } public static EndpointCertificateMetadata fromJsonString(String zkData) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 8b7590e88a9..3b5b01d16aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoAddress; @@ -26,9 +27,13 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfoBillingContact; import java.net.URI; import java.security.Principal; import java.security.PublicKey; +import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; /** * Slime serialization of {@link Tenant} sub-types. @@ -64,12 +69,15 @@ public class TenantSerializer { private static final String productCodeField = "productCode"; private static final String pemDeveloperKeysField = "pemDeveloperKeys"; private static final String tenantInfoField = "info"; + private static final String lastLoginInfoField = "lastLoginInfo"; public Slime toSlime(Tenant tenant) { Slime slime = new Slime(); Cursor tenantObject = slime.setObject(); tenantObject.setString(nameField, tenant.name().value()); tenantObject.setString(typeField, valueOf(tenant.type())); + tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli()); + toSlime(tenant.lastLoginInfo(), tenantObject.setObject(lastLoginInfoField)); switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; @@ -113,33 +121,43 @@ public class TenantSerializer { billingInfoObject.setString(productCodeField, billingInfo.productCode()); } - public Tenant tenantFrom(Slime slime) { + private void toSlime(LastLoginInfo lastLoginInfo, Cursor lastLoginInfoObject) { + for (LastLoginInfo.UserLevel userLevel: LastLoginInfo.UserLevel.values()) { + lastLoginInfo.get(userLevel).ifPresent(lastLoginAt -> + lastLoginInfoObject.setLong(valueOf(userLevel), lastLoginAt.toEpochMilli())); + } + } + + public Tenant tenantFrom(Slime slime, Supplier<Instant> tenantCreateTimeSupplier) { Inspector tenantObject = slime.get(); - Tenant.Type type; - type = typeOf(tenantObject.field(typeField).asString()); + Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); switch (type) { - case athenz: return athenzTenantFrom(tenantObject); - case cloud: return cloudTenantFrom(tenantObject); + case athenz: return athenzTenantFrom(tenantObject, tenantCreateTimeSupplier); + case cloud: return cloudTenantFrom(tenantObject, tenantCreateTimeSupplier); default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } } - private AthenzTenant athenzTenantFrom(Inspector tenantObject) { + private AthenzTenant athenzTenantFrom(Inspector tenantObject, Supplier<Instant> tenantCreateTimeSupplier) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); AthenzDomain domain = new AthenzDomain(tenantObject.field(athenzDomainField).asString()); Property property = new Property(tenantObject.field(propertyField).asString()); Optional<PropertyId> propertyId = SlimeUtils.optionalString(tenantObject.field(propertyIdField)).map(PropertyId::new); Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - return new AthenzTenant(name, domain, property, propertyId, contact); + Instant createdAt = SlimeUtils.optionalLong(tenantObject.field(createdAtField)).map(Instant::ofEpochMilli).orElseGet(tenantCreateTimeSupplier); + LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt, lastLoginInfo); } - private CloudTenant cloudTenantFrom(Inspector tenantObject) { + private CloudTenant cloudTenantFrom(Inspector tenantObject, Supplier<Instant> tenantCreateTimeSupplier) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); + Instant createdAt = SlimeUtils.optionalLong(tenantObject.field(createdAtField)).map(Instant::ofEpochMilli).orElseGet(tenantCreateTimeSupplier); + LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); Optional<Principal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); - return new CloudTenant(name, creator, developerKeys, info); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info); } private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) { @@ -182,6 +200,13 @@ public class TenantSerializer { .withAddress(tenantInfoAddressFromSlime(billingObject.field("address"))); } + private LastLoginInfo lastLoginInfoFromSlime(Inspector lastLoginInfoObject) { + Map<LastLoginInfo.UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(); + lastLoginInfoObject.traverse((String name, Inspector value) -> + lastLoginByUserLevel.put(userLevelOf(name), Instant.ofEpochMilli(value.asLong()))); + return new LastLoginInfo(lastLoginByUserLevel); + } + void toSlime(TenantInfo info, Cursor parentCursor) { if (info.isEmpty()) return; Cursor infoCursor = parentCursor.setObject("info"); @@ -279,4 +304,22 @@ public class TenantSerializer { default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } } + + private static LastLoginInfo.UserLevel userLevelOf(String value) { + switch (value) { + case "user": return LastLoginInfo.UserLevel.user; + case "developer": return LastLoginInfo.UserLevel.developer; + case "administrator": return LastLoginInfo.UserLevel.administrator; + default: throw new IllegalArgumentException("Unknown user level '" + value + "'."); + } + } + + private static String valueOf(LastLoginInfo.UserLevel userLevel) { + switch (userLevel) { + case user: return "user"; + case developer: return "developer"; + case administrator: return "administrator"; + default: throw new IllegalArgumentException("Unexpected user level '" + userLevel + "'."); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8378b914fe4..5a1496bf507 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -91,6 +91,7 @@ import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoAddress; @@ -1996,8 +1997,16 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<Instant> lastSubmission = applications.stream() .flatMap(app -> app.latestVersion().flatMap(ApplicationVersion::buildTime).stream()) .max(Comparator.naturalOrder()); + object.setLong("createdAtMillis", tenant.createdAt().toEpochMilli()); lastDev.ifPresent(instant -> object.setLong("lastDeploymentToDevMillis", instant.toEpochMilli())); lastSubmission.ifPresent(instant -> object.setLong("lastSubmissionToProdMillis", instant.toEpochMilli())); + + tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.user) + .ifPresent(instant -> object.setLong("lastLoginByUserMillis", instant.toEpochMilli())); + tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.developer) + .ifPresent(instant -> object.setLong("lastLoginByDeveloperMillis", instant.toEpochMilli())); + tenant.lastLoginInfo().get(LastLoginInfo.UserLevel.administrator) + .ifPresent(instant -> object.setLong("lastLoginByAdministratorMillis", instant.toEpochMilli())); } /** Returns a copy of the given URI with the host and port from the given URI, the path set to the given path and the query set to given query*/ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index b9cf5ca4f4d..3ca7e5ac249 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -1,11 +1,16 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; +import com.auth0.jwt.JWT; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; + +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.Date; import java.util.logging.Level; import com.yahoo.restapi.Path; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -64,9 +69,14 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { try { Principal principal = request.getUserPrincipal(); if (principal instanceof AthenzPrincipal) { + Instant issuedAt = request.getClientCertificateChain().stream().findFirst() + .map(X509Certificate::getNotBefore) + .or(() -> Optional.ofNullable((String) request.getAttribute("okta.access-token")).map(iat -> JWT.decode(iat).getIssuedAt())) + .map(Date::toInstant) + .orElse(Instant.EPOCH); request.setAttribute(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, - roles((AthenzPrincipal) principal, - request.getUri()))); + roles((AthenzPrincipal) principal, request.getUri()), + issuedAt)); } } catch (Exception e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java new file mode 100644 index 00000000000..9b1ccc09499 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilter.java @@ -0,0 +1,73 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.google.inject.Inject; +import com.yahoo.config.provision.TenantName; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.TenantController; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.api.role.TenantRole; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.administrator; +import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.developer; +import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.user; + +/** + * A security filter protects all controller apis. + * + * @author freva + */ +public class LastLoginUpdateFilter extends JsonSecurityRequestFilterBase { + + private static final Logger log = Logger.getLogger(LastLoginUpdateFilter.class.getName()); + + private final TenantController tenantController; + + @Inject + public LastLoginUpdateFilter(Controller controller) { + this.tenantController = controller.tenants(); + } + + @Override + public Optional<ErrorResponse> filter(DiscFilterRequest request) { + try { + SecurityContext context = (SecurityContext) request.getAttribute(SecurityContext.ATTRIBUTE_NAME); + Map<TenantName, List<LastLoginInfo.UserLevel>> userLevelsByTenant = context.roles().stream() + .flatMap(LastLoginUpdateFilter::filterTenantUserLevels) + .collect(Collectors.groupingBy(Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); + + userLevelsByTenant.forEach((tenant, userLevels) -> tenantController.updateLastLogin(tenant, userLevels, context.issuedAt())); + } catch (Exception e) { + log.log(Level.WARNING, "Exception updating last login:", e); + } + return Optional.empty(); + } + + public static Stream<Map.Entry<TenantName, LastLoginInfo.UserLevel>> filterTenantUserLevels(Role role) { + if (!(role instanceof TenantRole)) + return Stream.empty(); + + TenantRole tenantRole = (TenantRole) role; + TenantName name = tenantRole.tenant(); + switch (tenantRole.definition()) { + case athenzTenantAdmin: + return Stream.of(Map.entry(name, user), Map.entry(name, developer), Map.entry(name, administrator)); + case reader: return Stream.of(Map.entry(name, user)); + case developer: return Stream.of(Map.entry(name, developer)); + case administrator: return Stream.of(Map.entry(name, administrator)); + default: return Stream.empty(); + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 3be8d0cfe66..1c6511514a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -86,14 +86,14 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase { .map(CloudTenant.class::cast); if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key)) return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key), - Set.of(Role.reader(id.tenant()), - Role.developer(id.tenant())))); + Set.of(Role.reader(id.tenant()), Role.developer(id.tenant())), + controller.clock().instant())); Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id)); if (application.isPresent() && application.get().deployKeys().contains(key)) return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()), - Set.of(Role.reader(id.tenant()), - Role.headless(id.tenant(), id.application())))); + Set.of(Role.reader(id.tenant()), Role.headless(id.tenant(), id.application())), + controller.clock().instant())); } return Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java index 02387213135..32bb866a5ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import java.time.Instant; import java.util.List; /** @@ -22,11 +23,12 @@ public interface AccessControl { * Sets up access control based on the given credentials, and returns a tenant, based on the given specification. * * @param tenantSpec specification for the tenant to create + * @param createdAt instant when the tenant was created * @param credentials the credentials for the entity requesting the creation * @param existing list of existing tenants, to check for conflicts * @return the created tenant, for keeping */ - Tenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing); + Tenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing); /** * Modifies access control based on the given credentials, and returns a modified tenant, based on the given specification. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index d37e1e05030..563c230e4f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.ForbiddenException; +import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -51,12 +52,12 @@ public class CloudAccessControl implements AccessControl { } @Override - public CloudTenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { + public CloudTenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing) { requireTenantCreationAllowed((Auth0Credentials) credentials); requireTenantTrialLimitNotReached(existing); CloudTenantSpec spec = (CloudTenantSpec) tenantSpec; - CloudTenant tenant = CloudTenant.create(spec.tenant(), credentials.user()); + CloudTenant tenant = CloudTenant.create(spec.tenant(), createdAt, credentials.user()); for (Role role : Roles.tenantRoles(spec.tenant())) { userManagement.createRole(role); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java index 5682d8b69fe..7fa46031c98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -23,11 +24,11 @@ public class AthenzTenant extends Tenant { /** * This should only be used by serialization. - * Use {@link #create(TenantName, AthenzDomain, Property, Optional)}. + * Use {@link #create(TenantName, AthenzDomain, Property, Optional, Instant)}. * */ public AthenzTenant(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact) { - super(name, Objects.requireNonNull(contact, "contact must be non-null")); + Optional<Contact> contact, Instant createdAt, LastLoginInfo lastLoginInfo) { + super(name, createdAt, lastLoginInfo, contact); this.domain = Objects.requireNonNull(domain, "domain must be non-null"); this.property = Objects.requireNonNull(property, "property must be non-null"); this.propertyId = Objects.requireNonNull(propertyId, "propertyId must be non-null"); @@ -60,13 +61,8 @@ public class AthenzTenant extends Tenant { /** Create a new Athenz tenant */ public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property, - Optional<PropertyId> propertyId) { - return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty()); - } - - public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property, - Optional<PropertyId> propertyId, Optional<Contact> contact) { - return new AthenzTenant(requireName(name), domain, property, propertyId, contact); + Optional<PropertyId> propertyId, Instant createdAt) { + return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt, LastLoginInfo.EMPTY); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 67b285bb24f..5d0bb780c81 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.TenantName; import java.security.Principal; import java.security.PublicKey; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -22,16 +23,19 @@ public class CloudTenant extends Tenant { private final TenantInfo info; /** Public for the serialization layer — do not use! */ - public CloudTenant(TenantName name, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { - super(name, Optional.empty()); + public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, + BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { + super(name, createdAt, lastLoginInfo, Optional.empty()); this.creator = creator; this.developerKeys = developerKeys; this.info = Objects.requireNonNull(info); } /** Creates a tenant with the given name, provided it passes validation. */ - public static CloudTenant create(TenantName tenantName, Principal creator) { + public static CloudTenant create(TenantName tenantName, Instant createdAt, Principal creator) { return new CloudTenant(requireName(tenantName), + createdAt, + LastLoginInfo.EMPTY, Optional.ofNullable(creator), ImmutableBiMap.of(), TenantInfo.EMPTY); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java new file mode 100644 index 00000000000..15f2f97e7d1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/LastLoginInfo.java @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.tenant; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * @author freva + */ +public class LastLoginInfo { + + public static final LastLoginInfo EMPTY = new LastLoginInfo(Map.of()); + + private final Map<UserLevel, Instant> lastLoginByUserLevel; + + public LastLoginInfo(Map<UserLevel, Instant> lastLoginByUserLevel) { + this.lastLoginByUserLevel = Map.copyOf(lastLoginByUserLevel); + } + + public Optional<Instant> get(UserLevel userLevel) { + return Optional.ofNullable(lastLoginByUserLevel.get(userLevel)); + } + + /** + * Returns new instance with updated last login time if the given {@code loginAt} timestamp is after the current + * for the given {@code userLevel}, otherwise returns this + */ + public LastLoginInfo withLastLoginIfLater(UserLevel userLevel, Instant loginAt) { + Instant lastLogin = lastLoginByUserLevel.getOrDefault(userLevel, Instant.EPOCH); + if (loginAt.isAfter(lastLogin)) { + Map<UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(this.lastLoginByUserLevel); + lastLoginByUserLevel.put(userLevel, loginAt); + return new LastLoginInfo(lastLoginByUserLevel); + } + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + LastLoginInfo lastLoginInfo = (LastLoginInfo) o; + return lastLoginByUserLevel.equals(lastLoginInfo.lastLoginByUserLevel); + } + + @Override + public int hashCode() { + return lastLoginByUserLevel.hashCode(); + } + + public enum UserLevel { user, developer, administrator }; +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java index b1dc0d8a5d5..f8b54e7eff3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -15,10 +16,14 @@ import java.util.Optional; public abstract class Tenant { private final TenantName name; + private final Instant createdAt; + private final LastLoginInfo lastLoginInfo; private final Optional<Contact> contact; - Tenant(TenantName name, Optional<Contact> contact) { + Tenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Contact> contact) { this.name = name; + this.createdAt = createdAt; + this.lastLoginInfo = lastLoginInfo; this.contact = contact; } @@ -27,6 +32,16 @@ public abstract class Tenant { return name; } + /** Instant when the tenant was created */ + public Instant createdAt() { + return createdAt; + } + + /** Returns login information for this tenant */ + public LastLoginInfo lastLoginInfo() { + return lastLoginInfo; + } + /** Contact information for this tenant */ public Optional<Contact> contact() { return contact; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index 1e3771dedb0..4f2000b1902 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -114,7 +114,7 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); - assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()); + assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans()); } @Test @@ -124,12 +124,18 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); - assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()); + assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans()); } @Test public void reuses_stored_certificate_metadata() { - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0)); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id", + List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + "default.default.global.vespa.oath.cloud", + "*.default.default.global.vespa.oath.cloud", + "default.default.aws-us-east-1a.vespa.oath.cloud", + "*.default.default.aws-us-east-1a.vespa.oath.cloud"), + "", Optional.empty(), Optional.empty())); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); @@ -146,7 +152,13 @@ public class EndpointCertificateManagerTest { secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8); - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0)); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id", + List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + "default.default.global.vespa.oath.cloud", + "*.default.default.global.vespa.oath.cloud", + "default.default.aws-us-east-1a.vespa.oath.cloud", + "*.default.default.aws-us-east-1a.vespa.oath.cloud"), + "issuer", Optional.empty(), Optional.empty())); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); @@ -156,7 +168,7 @@ public class EndpointCertificateManagerTest { @Test public void reprovisions_certificate_when_necessary() { - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, Optional.of("uuid"), Optional.of(List.of()), Optional.empty())); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); @@ -169,7 +181,7 @@ public class EndpointCertificateManagerTest { public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, Optional.of("uuid"), Optional.of(expectedSans), Optional.of("mockCa"))); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", expectedSans, "mockCa", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1); @@ -180,7 +192,7 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(0, endpointCertificateMetadata.get().version()); assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow())); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans())); } @Test @@ -202,6 +214,6 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow())); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java new file mode 100644 index 00000000000..dbf102f23d7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -0,0 +1,126 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author andreer + */ +public class EndpointCertificateMaintainerTest { + + private final ControllerTester tester = new ControllerTester(); + private final SecretStoreMock secretStore = (SecretStoreMock) tester.controller().secretStore(); + private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1)); + private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty()); + + @Before + public void setUp() throws Exception { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.id(), true); + } + + @Test + public void old_and_unused_cert_is_deleted() { + tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); + assertTrue(maintainer.maintain()); + assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty()); + } + + @Test + public void unused_but_recently_used_cert_is_not_deleted() { + EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); + tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); + assertTrue(maintainer.maintain()); + assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); + } + + @Test + public void refreshed_certificate_is_updated() { + EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); + tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); + + secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); + secretStore.setSecret(exampleMetadata.certName(), "bar", 1); + + assertTrue(maintainer.maintain()); + + var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1)); + + assertEquals(updatedCert, tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); + } + + @Test + public void certificate_in_use_is_not_deleted() { + var appId = ApplicationId.from("tenant", "application", "default"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + + var applicationPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + + DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); + + deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); + + + tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata); + + assertTrue(maintainer.maintain()); + assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed! + } + + @Test + public void refreshed_certificate_is_deployed_after_one_week() { + var appId = ApplicationId.from("tenant", "application", "default"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + + var applicationPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + + DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); + + deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); + + tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata); + + assertTrue(maintainer.maintain()); + assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed! + + tester.clock().advance(Duration.ofDays(3)); + + secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); + secretStore.setSecret(exampleMetadata.certName(), "bar", 1); + + maintainer.maintain(); + + tester.clock().advance(Duration.ofDays(8)); + + deploymentContext.assertNotRunning(productionUsWest1); + + maintainer.maintain(); + + deploymentContext.assertRunning(productionUsWest1); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 9c6790f630b..00f5335bd82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -10,46 +10,39 @@ import static org.junit.Assert.*; public class EndpointCertificateMetadataSerializerTest { - private final EndpointCertificateMetadata sample = - new EndpointCertificateMetadata("keyName", "certName", 1, 0); - private final EndpointCertificateMetadata sampleWithRequestMetadata = - new EndpointCertificateMetadata("keyName", "certName", 1, 0, Optional.of("requestId"), Optional.of(List.of("SAN1", "SAN2")), Optional.of("issuer")); + private final EndpointCertificateMetadata sampleWithExpiryAndLastRefreshed = + new EndpointCertificateMetadata("keyName", "certName", 1, 0, "requestId", List.of("SAN1", "SAN2"), "issuer", java.util.Optional.of(1628000000L), Optional.of(1612000000L)); + + private final EndpointCertificateMetadata sampleWithoutExpiry = + new EndpointCertificateMetadata("keyName", "certName", 1, 0, "requestId", List.of("SAN1", "SAN2"), "issuer", Optional.empty(), Optional.empty()); @Test - public void serialize() { + public void serializeWithExpiryAndLastRefreshed() { assertEquals( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0}", - EndpointCertificateMetadataSerializer.toSlime(sample).toString()); + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\",\"expiry\":1628000000,\"lastRefreshed\":1612000000}", + EndpointCertificateMetadataSerializer.toSlime(sampleWithExpiryAndLastRefreshed).toString()); } @Test - public void serializeWithRequestMetadata() { + public void serializeWithoutExpiryAndLastRefreshed() { assertEquals( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}", - EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString()); + EndpointCertificateMetadataSerializer.toSlime(sampleWithoutExpiry).toString()); } @Test - public void deserializeFromJson() { + public void deserializeFromJsonWithExpiryAndLastRefreshed() { assertEquals( - sample, + sampleWithExpiryAndLastRefreshed, EndpointCertificateMetadataSerializer.fromJsonString( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0}")); + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\",\"expiry\":1628000000,\"lastRefreshed\":1612000000}")); } @Test - public void deserializeFromJsonWithRequestMetadata() { + public void deserializeFromJsonWithoutExpiryAndLastRefreshed() { assertEquals( - sampleWithRequestMetadata, + sampleWithoutExpiry, EndpointCertificateMetadataSerializer.fromJsonString( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}")); } - - @Test - public void deserializeFromJsonWithDefaultLastRequested() { - assertEquals( - new EndpointCertificateMetadata("keyName", "certName", 1, 1597200000), - EndpointCertificateMetadataSerializer.fromJsonString( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 20caceee097..4fcf4f344e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoAddress; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoBillingContact; @@ -20,8 +21,10 @@ import org.junit.Test; import java.net.URI; import java.security.PublicKey; -import java.util.Collections; +import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -48,13 +51,15 @@ public class TenantSerializerTest { AthenzTenant tenant = AthenzTenant.create(TenantName.from("athenz-tenant"), new AthenzDomain("domain1"), new Property("property1"), - Optional.of(new PropertyId("1"))); - AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + Optional.of(new PropertyId("1")), + Instant.ofEpochMilli(1234L)); + AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.domain(), serialized.domain()); assertEquals(tenant.property(), serialized.property()); assertTrue(serialized.propertyId().isPresent()); assertEquals(tenant.propertyId(), serialized.propertyId()); + assertEquals(tenant.createdAt(), serialized.createdAt()); } @Test @@ -62,8 +67,9 @@ public class TenantSerializerTest { AthenzTenant tenant = AthenzTenant.create(TenantName.from("athenz-tenant"), new AthenzDomain("domain1"), new Property("property1"), - Optional.empty()); - AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + Optional.empty(), + Instant.EPOCH); + AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertFalse(serialized.propertyId().isPresent()); assertEquals(tenant.propertyId(), serialized.propertyId()); } @@ -74,32 +80,39 @@ public class TenantSerializerTest { new AthenzDomain("domain1"), new Property("property1"), Optional.of(new PropertyId("1")), - Optional.of(contact())); - AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + Optional.of(contact()), + Instant.EPOCH, + lastLoginInfo(321L, 654L, 987L)); + AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.contact(), serialized.contact()); } @Test public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.ofEpochMilli(1234L), + lastLoginInfo(123L, 456L, null), Optional.of(new SimplePrincipal("foobar-user")), ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), otherPublicKey, new SimplePrincipal("jane")), TenantInfo.EMPTY); - CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.creator(), serialized.creator()); assertEquals(tenant.developerKeys(), serialized.developerKeys()); + assertEquals(tenant.createdAt(), serialized.createdAt()); } @Test public void cloud_tenant_with_info() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.EPOCH, + lastLoginInfo(null, 789L, 654L), Optional.of(new SimplePrincipal("foobar-user")), ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), otherPublicKey, new SimplePrincipal("jane")), TenantInfo.EMPTY.withName("Ofni Tnanet")); - CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.info(), serialized.info()); } @@ -148,18 +161,25 @@ public class TenantSerializerTest { assertEquals(fullInfo, roundTripInfo); } - private Contact contact() { + private static Contact contact() { return new Contact( URI.create("http://contact1.test"), URI.create("http://property1.test"), URI.create("http://issue-tracker-1.test"), List.of( - Collections.singletonList("person1"), - Collections.singletonList("person2") + List.of("person1"), + List.of("person2") ), "queue", Optional.empty() ); } + private static LastLoginInfo lastLoginInfo(Long user, Long developer, Long administrator) { + Map<LastLoginInfo.UserLevel, Instant> lastLogins = new HashMap<>(); + Optional.ofNullable(user).map(Instant::ofEpochMilli).ifPresent(i -> lastLogins.put(LastLoginInfo.UserLevel.user, i)); + Optional.ofNullable(developer).map(Instant::ofEpochMilli).ifPresent(i -> lastLogins.put(LastLoginInfo.UserLevel.developer, i)); + Optional.ofNullable(administrator).map(Instant::ofEpochMilli).ifPresent(i -> lastLogins.put(LastLoginInfo.UserLevel.administrator, i)); + return new LastLoginInfo(lastLogins); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e5f11beb9a2..434c83898ee 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -67,6 +67,7 @@ import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; @@ -92,7 +93,6 @@ import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static java.net.URLEncoder.encode; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.joining; @@ -141,7 +141,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final UserId OTHER_USER_ID = new UserId("otheruser"); private static final UserId HOSTED_VESPA_OPERATOR = new UserId("johnoperator"); private static final OktaIdentityToken OKTA_IT = new OktaIdentityToken("okta-it"); - private static final OktaAccessToken OKTA_AT = new OktaAccessToken("okta-at"); + private static final OktaAccessToken OKTA_AT = new OktaAccessToken("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.he0ErCNloe4J7Id0Ry2SEDg09lKkZkfsRiGsdX_vgEg"); private ContainerTester tester; @@ -193,8 +193,10 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("tenant-without-applications-with-id.json")); // GET a tenant with property ID and contact information updateContactInformation(); + tester.controller().tenants().updateLastLogin(TenantName.from("tenant2"), + List.of(LastLoginInfo.UserLevel.user, LastLoginInfo.UserLevel.administrator), Instant.ofEpochMilli(1234)); tester.assertResponse(request("/application/v4/tenant/tenant2", GET).userIdentity(USER_ID), - new File("tenant-with-contact-info.json")); + new File("tenant2.json")); // POST (create) an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) @@ -1236,9 +1238,10 @@ public class ApplicationApiTest extends ControllerContainerTest { accessDenied, 403); - // Create legancy tenant name containing underscores + // Create legacy tenant name containing underscores tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, - new Property("property1"), Optional.empty(), Optional.empty())); + new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH, LastLoginInfo.EMPTY)); + // POST (add) a Athenz tenant with dashes duplicates existing one with underscores tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .userIdentity(USER_ID) @@ -1339,7 +1342,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"athensDomain\":\"domain2\", \"property\":\"property1\"}") .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"tenant\":\"tenant1\",\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[],\"metaData\":{}}", + new File("tenant1.json"), 200); // Deleting a tenant for an Athens domain the user is not admin for is disallowed diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json index b11b65ead30..194f2cdd494 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json @@ -11,7 +11,8 @@ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ], - "metaData":{ + "metaData": { + "createdAtMillis": "(ignore)", "lastDeploymentToDevMillis":"(ignore)", "lastSubmissionToProdMillis":1000 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index 578323be373..b02b1ea2565 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -11,5 +11,7 @@ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ], - "metaData":{} + "metaData": { + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json deleted file mode 100644 index a7d1d7413f3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "tenant": "tenant2", - "type": "ATHENS", - "athensDomain": "domain2", - "property": "property2", - "propertyId": "1234", - "propertyUrl": "www.properties.tld/1234", - "contactsUrl": "www.contacts.tld/1234", - "issueCreationUrl": "www.issues.tld/1234", - "contacts": [ - [ - "alice" - ], - [ - "bob" - ] - ], - "applications": [], - "metaData":{} -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json index 5e8f0e4b575..bd77a68a1eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json @@ -5,5 +5,7 @@ "property": "property2", "propertyId": "1234", "applications": [], - "metaData": {} + "metaData": { + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json index 82848fe971d..33ed505ce35 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json @@ -3,8 +3,8 @@ "type": "ATHENS", "athensDomain": "domain1", "property": "property1", - "applications": [ - - ], - "metaData":{} + "applications": [ ], + "metaData":{ + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json index 574c3d2c476..551b26c8513 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json @@ -6,5 +6,9 @@ "applications": [ @include(instance1-recursive.json) ], - "metaData":{"lastDeploymentToDevMillis":"(ignore)","lastSubmissionToProdMillis":1000} + "metaData": { + "createdAtMillis": "(ignore)", + "lastDeploymentToDevMillis": "(ignore)", + "lastSubmissionToProdMillis": 1000 + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json new file mode 100644 index 00000000000..a105a194974 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json @@ -0,0 +1,10 @@ +{ + "tenant": "tenant1", + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property1", + "applications": [], + "metaData": { + "createdAtMillis": "(ignore)" + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json index 21d5ceba805..497d80c96a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json @@ -16,5 +16,9 @@ ] ], "applications": [], - "metaData":{} + "metaData": { + "createdAtMillis": "(ignore)", + "lastLoginByUserMillis": 1234, + "lastLoginByAdministratorMillis": 1234 + } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 5af91e17bb7..6f67b0d8aa8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -31,6 +31,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "EndpointCertificateMaintainer" + }, + { "name": "HostSwitchUpdater" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java new file mode 100644 index 00000000000..df402e8c594 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/LastLoginUpdateFilterTest.java @@ -0,0 +1,59 @@ +package com.yahoo.vespa.hosted.controller.restapi.filter; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; +import org.junit.Test; + +import java.time.Instant; +import java.util.Set; + +import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.administrator; +import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.developer; +import static com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo.UserLevel.user; + +import static org.junit.Assert.assertEquals; + +public class LastLoginUpdateFilterTest { + + private static final TenantName tenant1 = TenantName.from("tenant1"); + private static final TenantName tenant2 = TenantName.from("tenant2"); + + private final ControllerTester tester = new ControllerTester(); + private final LastLoginUpdateFilter filter = new LastLoginUpdateFilter(tester.controller()); + + @Test + public void updateLastLoginTimeTest() { + tester.createTenant(tenant1.value()); + tester.createTenant(tenant2.value()); + + request(123, Role.developer(tenant1), Role.reader(tenant1), Role.athenzTenantAdmin(tenant2)); + assertLastLoginBy(tenant1, 123L, 123L, null); + assertLastLoginBy(tenant2, 123L, 123L, 123L); + + request(321, Role.administrator(tenant1), Role.reader(tenant1)); + assertLastLoginBy(tenant1, 321L, 123L, 321L); + assertLastLoginBy(tenant2, 123L, 123L, 123L); + } + + private void assertLastLoginBy(TenantName tenantName, Long lastUserLoginAt, Long lastDeveloperLoginAt, Long lastAdministratorLoginAt) { + LastLoginInfo loginInfo = tester.controller().tenants().require(tenantName).lastLoginInfo(); + assertEquals(lastUserLoginAt, loginInfo.get(user).map(Instant::toEpochMilli).orElse(null)); + assertEquals(lastDeveloperLoginAt, loginInfo.get(developer).map(Instant::toEpochMilli).orElse(null)); + assertEquals(lastAdministratorLoginAt, loginInfo.get(administrator).map(Instant::toEpochMilli).orElse(null)); + } + + private void request(long issuedAt, Role... roles) { + SecurityContext context = new SecurityContext(() -> "bob", Set.of(roles), Instant.ofEpochMilli(issuedAt)); + Request request = new Request("/", new byte[0], Request.Method.GET, context.principal()); + request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, context); + filter.filter(new ApplicationRequestToDiscFilterRequestWrapper(request)); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 26c2e2d1175..390823271b4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import org.junit.Before; import org.junit.Test; @@ -27,6 +28,7 @@ import java.net.URI; import java.net.http.HttpRequest; import java.security.PrivateKey; import java.security.PublicKey; +import java.time.Instant; import java.util.Optional; import java.util.Set; @@ -67,6 +69,8 @@ public class SignatureFilterTest { signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); tester.curator().writeTenant(new CloudTenant(appId.tenant(), + Instant.EPOCH, + LastLoginInfo.EMPTY, Optional.empty(), ImmutableBiMap.of(), TenantInfo.EMPTY)); @@ -96,24 +100,29 @@ public class SignatureFilterTest { verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), Set.of(Role.reader(id.tenant()), - Role.headless(id.tenant(), id.application())))); + Role.headless(id.tenant(), id.application())), + tester.clock().instant())); // Signed POST request with X-Key header gets a headless role. byte[] hiBytes = new byte[]{0x48, 0x69}; verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), Set.of(Role.reader(id.tenant()), - Role.headless(id.tenant(), id.application())))); + Role.headless(id.tenant(), id.application())), + tester.clock().instant())); // Signed request gets a developer role when a matching developer key is stored for the tenant. tester.curator().writeTenant(new CloudTenant(appId.tenant(), + Instant.EPOCH, + LastLoginInfo.EMPTY, Optional.empty(), ImmutableBiMap.of(publicKey, () -> "user"), TenantInfo.EMPTY)); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), Set.of(Role.reader(id.tenant()), - Role.developer(id.tenant())))); + Role.developer(id.tenant())), + tester.clock().instant())); // Unsigned requests still get no roles. verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 28dad0b0c2f..9323067904c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -16,5 +16,7 @@ "clusterSize": 5 }, "applications": [], - "metaData":{} + "metaData":{ + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index 73b97e09827..eaabb9fe3e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -9,5 +9,7 @@ "clusterSize": 5 }, "applications": [], - "metaData":{} + "metaData":{ + "createdAtMillis": "(ignore)" + } } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index d8bbf21605a..d95d8ac3352 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -1,4 +1,4 @@ -# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. include(VespaExtendedDefaultBuildSettings OPTIONAL) @@ -89,11 +89,16 @@ function(setup_vespa_default_build_settings_ubuntu_20_04) set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_debian_10) - message("-- Setting up default build settings for debian 10") +function(setup_vespa_default_build_settings_debian) + message("-- Setting up default build settings for debian") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" PARENT_SCOPE) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) + SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) + SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) + find_package(LLVM REQUIRED CONFIG) + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + set(DEFAULT_VESPA_LLVM_VERSION ${LLVM_VERSION_MAJOR} PARENT_SCOPE) endfunction() function(vespa_use_default_vespa_unprivileged) @@ -183,13 +188,13 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_ubuntu_19_10() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 20.04") setup_vespa_default_build_settings_ubuntu_20_04() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "debian 10") - setup_vespa_default_build_settings_debian_10() + elseif(VESPA_OS_DISTRO STREQUAL "debian") + setup_vespa_default_build_settings_debian() else() - message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") + message(FATAL_ERROR "-- Unknown vespa build platform ${VESPA_OS_DISTRO_COMBINED}") endif() if(NOT DEFINED VESPA_LLVM_VERSION AND NOT DEFINED DEFAULT_VESPA_LLVM_VERSION) - message(FATAL_ERROR "-- Unkonwn default llvm version") + message(FATAL_ERROR "-- Unknown default llvm version") endif() if(NOT DEFINED DEFAULT_CMAKE_PREFIX_PATH) set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}") diff --git a/documentgen-test/pom.xml b/documentgen-test/pom.xml index 52986be7465..72fbe1682a7 100644 --- a/documentgen-test/pom.xml +++ b/documentgen-test/pom.xml @@ -27,6 +27,12 @@ <scope>compile</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <!-- TODO: Excluded from container-dev. Remove when deps are explicitly listed. --> <groupId>com.yahoo.vespa</groupId> <artifactId>linguistics</artifactId> diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 00618f2c838..8d6f48adb21 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -32,12 +32,25 @@ vespa_define_module( src/tests/eval/tensor_function src/tests/eval/tensor_lambda src/tests/eval/tensor_spec + src/tests/eval/typed_cells src/tests/eval/value_cache src/tests/eval/value_codec src/tests/eval/value_type - src/tests/eval/typed_cells src/tests/gp/ponder_nov2017 + src/tests/instruction/add_trivial_dimension_optimizer + src/tests/instruction/dense_dot_product_function + src/tests/instruction/dense_inplace_join_function + src/tests/instruction/dense_matmul_function + src/tests/instruction/dense_multi_matmul_function + src/tests/instruction/dense_replace_type_function + src/tests/instruction/dense_simple_expand_function + src/tests/instruction/dense_simple_join_function + src/tests/instruction/dense_simple_map_function + src/tests/instruction/dense_single_reduce_function + src/tests/instruction/dense_tensor_create_function + src/tests/instruction/dense_tensor_peek_function src/tests/instruction/dense_xw_product_function + src/tests/instruction/fast_rename_optimizer src/tests/instruction/generic_concat src/tests/instruction/generic_create src/tests/instruction/generic_join @@ -46,23 +59,11 @@ vespa_define_module( src/tests/instruction/generic_peek src/tests/instruction/generic_reduce src/tests/instruction/generic_rename - src/tests/instruction/dense_dot_product_function - src/tests/instruction/dense_matmul_function - src/tests/instruction/dense_multi_matmul_function - src/tests/instruction/dense_simple_expand_function - src/tests/instruction/dense_tensor_peek_function src/tests/instruction/index_lookup_table src/tests/instruction/join_with_number - src/tests/instruction/add_trivial_dimension_optimizer - src/tests/instruction/fast_rename_optimizer - src/tests/instruction/dense_inplace_join_function - src/tests/instruction/dense_pow_as_map_optimizer + src/tests/instruction/mixed_inner_product_function + src/tests/instruction/pow_as_map_optimizer src/tests/instruction/remove_trivial_dimension_optimizer - src/tests/instruction/dense_replace_type_function - src/tests/instruction/dense_simple_join_function - src/tests/instruction/dense_simple_map_function - src/tests/instruction/dense_single_reduce_function - src/tests/instruction/dense_tensor_create_function src/tests/instruction/vector_from_doubles_function src/tests/streamed/value src/tests/tensor/instruction_benchmark diff --git a/eval/src/tests/eval/aggr/aggr_test.cpp b/eval/src/tests/eval/aggr/aggr_test.cpp index 9045df68305..5eddb026406 100644 --- a/eval/src/tests/eval/aggr/aggr_test.cpp +++ b/eval/src/tests/eval/aggr/aggr_test.cpp @@ -85,8 +85,9 @@ TEST("require that PROD aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::PROD); } -TEST("require that Prod combine works as expected") { +TEST("require that Prod static API works as expected") { using Type = Prod<double>; + EXPECT_EQUAL(Type::null_value(), 1.0); EXPECT_EQUAL(Type::combine(3,7), 21.0); EXPECT_EQUAL(Type::combine(5,4), 20.0); } @@ -103,8 +104,9 @@ TEST("require that SUM aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::SUM); } -TEST("require that Sum combine works as expected") { +TEST("require that Sum static API works as expected") { using Type = Sum<double>; + EXPECT_EQUAL(Type::null_value(), 0.0); EXPECT_EQUAL(Type::combine(3,7), 10.0); EXPECT_EQUAL(Type::combine(5,4), 9.0); } @@ -121,8 +123,10 @@ TEST("require that MAX aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::MAX); } -TEST("require that Max combine works as expected") { +TEST("require that Max static API works as expected") { using Type = Max<double>; + EXPECT_EQUAL(Max<double>::null_value(), -std::numeric_limits<double>::infinity()); + EXPECT_EQUAL(Max<float>::null_value(), -std::numeric_limits<float>::infinity()); EXPECT_EQUAL(Type::combine(3,7), 7.0); EXPECT_EQUAL(Type::combine(5,4), 5.0); } @@ -165,8 +169,10 @@ TEST("require that MIN aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::MIN); } -TEST("require that Min combine works as expected") { +TEST("require that Min static API works as expected") { using Type = Min<double>; + EXPECT_EQUAL(Min<double>::null_value(), std::numeric_limits<double>::infinity()); + EXPECT_EQUAL(Min<float>::null_value(), std::numeric_limits<float>::infinity()); EXPECT_EQUAL(Type::combine(3,7), 3.0); EXPECT_EQUAL(Type::combine(5,4), 4.0); } diff --git a/eval/src/tests/instruction/dense_pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/instruction/dense_pow_as_map_optimizer/CMakeLists.txt deleted file mode 100644 index d6ce9f1924c..00000000000 --- a/eval/src/tests/instruction/dense_pow_as_map_optimizer/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_dense_pow_as_map_optimizer_test_app TEST - SOURCES - dense_pow_as_map_optimizer_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_dense_pow_as_map_optimizer_test_app COMMAND eval_dense_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp index 3ab971dd34d..9e2090fa968 100644 --- a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp +++ b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp @@ -32,7 +32,8 @@ std::vector<Layout> layouts = { float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), {x(3),y({"foo", "bar"}),z(7)}, {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}), + {x(3),y({}),z(7)} }; TensorSpec perform_generic_reduce(const TensorSpec &a, Aggr aggr, const std::vector<vespalib::string> &dims, @@ -69,7 +70,9 @@ TEST(GenericReduceTest, sparse_reduce_plan_can_be_created) { void test_generic_reduce_with(const ValueBuilderFactory &factory) { for (const Layout &layout: layouts) { TensorSpec input = spec(layout, Div16(N())); + SCOPED_TRACE(fmt("tensor type: %s, num_cells: %zu", input.type().c_str(), input.cells().size())); for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { + SCOPED_TRACE(fmt("aggregator: %s", AggrNames::name_of(aggr)->c_str())); for (const Domain &domain: layout) { auto expect = ReferenceOperations::reduce(input, aggr, {domain.dimension}).normalize(); auto actual = perform_generic_reduce(input, aggr, {domain.dimension}, factory); diff --git a/eval/src/tests/instruction/mixed_inner_product_function/CMakeLists.txt b/eval/src/tests/instruction/mixed_inner_product_function/CMakeLists.txt new file mode 100644 index 00000000000..16f59dc28ed --- /dev/null +++ b/eval/src/tests/instruction/mixed_inner_product_function/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_mixed_inner_product_function_test_app TEST + SOURCES + mixed_inner_product_function_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_mixed_inner_product_function_test_app COMMAND eval_mixed_inner_product_function_test_app) diff --git a/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp b/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp new file mode 100644 index 00000000000..fbe71f3ed63 --- /dev/null +++ b/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp @@ -0,0 +1,155 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/instruction/dense_dot_product_function.h> +#include <vespa/eval/instruction/dense_matmul_function.h> +#include <vespa/eval/instruction/dense_multi_matmul_function.h> +#include <vespa/eval/instruction/dense_xw_product_function.h> +#include <vespa/eval/instruction/mixed_inner_product_function.h> +#include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("mixed_inner_product_function_test"); + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); + +struct MyVecSeq : Sequence { + double bias; + double operator[](size_t i) const override { return (i + bias); } + MyVecSeq(double cellBias) : bias(cellBias) {} +}; + +std::function<double(size_t)> my_vec_gen(double cellBias) { + return [=] (size_t i) noexcept { return i + cellBias; }; +} + +//----------------------------------------------------------------------------- + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add_vector("x", 3, my_vec_gen(2.0)) + .add_vector("x", 3, my_vec_gen(13.25)) + .add_vector("y", 3, my_vec_gen(4.0)) + .add_vector("z", 3, my_vec_gen(0.25)) + .add_matrix("x", 3, "y", 1, my_vec_gen(5.0)) + .add_matrix("x", 1, "y", 3, my_vec_gen(6.0)) + .add_matrix("x", 3, "y", 3, my_vec_gen(1.5)) + .add_matrix("x", 3, "z", 3, my_vec_gen(2.5)) + .add_cube("x", 3, "y", 3, "z", 3, my_vec_gen(-4.0)) + .add("mix_x3zm", spec({x(3),z({"c","d"})}, MyVecSeq(0.5))) + .add("mix_y3zm", spec({y(3),z({"c","d"})}, MyVecSeq(3.5))) + .add("mix_x3zm_f", spec(float_cells({x(3),z({"c","d"})}), MyVecSeq(0.5))) + .add("mix_y3zm_f", spec(float_cells({y(3),z({"c","d"})}), MyVecSeq(3.5))) + .add("mix_x3y3zm", spec({x(3),y(3),z({"c","d"})}, MyVecSeq(0.0))) + ; + +} +EvalFixture::ParamRepo param_repo = make_params(); + +void assert_mixed_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_factory, expr, param_repo, false); + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EXPECT_EQ(slow_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + auto info = fast_fixture.find_all<MixedInnerProductFunction>(); + ASSERT_EQ(info.size(), 1u); + EXPECT_TRUE(info[0]->result_is_mutable()); +} + +void assert_not_mixed_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_factory, expr, param_repo, false); + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EXPECT_EQ(slow_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + auto info = fast_fixture.find_all<MixedInnerProductFunction>(); + ASSERT_EQ(info.size(), 0u); +} + +void assert_dense_optimized(const vespalib::string &expr) { + EvalFixture slow_fixture(prod_factory, expr, param_repo, false); + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EXPECT_EQ(slow_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + auto info = fast_fixture.find_all<MixedInnerProductFunction>(); + ASSERT_EQ(info.size(), 0u); + auto info2 = fast_fixture.find_all<DenseDotProductFunction>(); + auto info3 = fast_fixture.find_all<DenseMatMulFunction>(); + auto info4 = fast_fixture.find_all<DenseMultiMatMulFunction>(); + auto info5 = fast_fixture.find_all<DenseXWProductFunction>(); + ASSERT_EQ(info2.size() + info3.size() + info4.size() + info5.size(), 1u); +} + +//----------------------------------------------------------------------------- + +TEST(MixedInnerProduct, use_dense_optimizers_when_possible) { + // actually, all these trigger DenseXWProduct + assert_dense_optimized("reduce(x3 * x3y1,sum,x)"); + assert_dense_optimized("reduce(y3 * x1y3,sum,y)"); + assert_dense_optimized("reduce(y3 * x3y3,sum,y)"); + assert_dense_optimized("reduce(x1y3 * y3,sum,y)"); + assert_dense_optimized("reduce(x3y3 * y3,sum,y)"); +} + +TEST(MixedInnerProduct, trigger_optimizer_when_possible) { + assert_mixed_optimized("reduce(x3 * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3f * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3 * mix_x3zm_f,sum,x)"); + assert_mixed_optimized("reduce(x3f * mix_x3zm_f,sum,x)"); + assert_mixed_optimized("reduce(x3$2 * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3f$2 * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(y3 * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(y3f * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(y3 * mix_y3zm_f,sum,y)"); + assert_mixed_optimized("reduce(y3f * mix_y3zm_f,sum,y)"); + assert_mixed_optimized("reduce(x3y1 * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3y1f * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3y1 * mix_x3zm,sum,x,y)"); + assert_mixed_optimized("reduce(x3y1f * mix_x3zm,sum,x,y)"); + assert_mixed_optimized("reduce(x1y3 * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(x1y3f * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(x1y3 * x1y3,sum,y)"); + assert_mixed_optimized("reduce(x1y3 * x1y3f,sum,y)"); + assert_mixed_optimized("reduce(x1y3f * x1y3,sum,y)"); + assert_mixed_optimized("reduce(x1y3f * x1y3f,sum,y)"); + assert_mixed_optimized("reduce(x1y3 * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(x1y3f * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(mix_x3zm * x3,sum,x)"); + assert_mixed_optimized("reduce(mix_x3zm * x3f,sum,x)"); + assert_mixed_optimized("reduce(mix_x3zm * x3y1,sum,x)"); + assert_mixed_optimized("reduce(mix_x3zm * x3y1f,sum,x)"); + assert_mixed_optimized("reduce(mix_y3zm * y3,sum,y)"); + assert_mixed_optimized("reduce(mix_y3zm * y3f,sum,y)"); + assert_mixed_optimized("reduce(mix_y3zm * x1y3,sum,y)"); + assert_mixed_optimized("reduce(mix_y3zm * x1y3f,sum,y)"); +} + +TEST(MixedInnerProduct, should_not_trigger_optimizer_for_other_cases) { + assert_not_mixed_optimized("reduce(x3y3z3 * x3,sum,x)"); + assert_not_mixed_optimized("reduce(x3y3z3 * y3,sum,y)"); + assert_not_mixed_optimized("reduce(x3y3z3 * x3y3,sum,x,y)"); + assert_not_mixed_optimized("reduce(x3y3 * mix_y3zm,sum,y)"); + assert_not_mixed_optimized("reduce(mix_y3zm * x3,sum,x,y)"); + assert_not_mixed_optimized("reduce(mix_x3y3zm * y3,sum,y,z)"); + assert_not_mixed_optimized("reduce(mix_x3y3zm * y3,sum,x,y)"); +} + +TEST(MixedInnerProduct, check_compatibility_with_complex_types) { + ValueType vec_type = ValueType::from_spec("tensor<float>(f[1],g[2],i[1],x[3],y[1])"); + ValueType mix_type = ValueType::from_spec("tensor<double>(cat{},g[2],host{},k[1],x[3],z{})"); + ValueType res_type = ValueType::join(vec_type,mix_type).reduce({"g","k","i","x"}); + EXPECT_EQ(MixedInnerProductFunction::compatible_types(res_type, mix_type, vec_type), true); + EXPECT_EQ(MixedInnerProductFunction::compatible_types(res_type, vec_type, mix_type), false); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/instruction/pow_as_map_optimizer/CMakeLists.txt new file mode 100644 index 00000000000..25eeb73df37 --- /dev/null +++ b/eval/src/tests/instruction/pow_as_map_optimizer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_pow_as_map_optimizer_test_app TEST + SOURCES + pow_as_map_optimizer_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_pow_as_map_optimizer_test_app COMMAND eval_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/instruction/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp index 67567b4e289..920198f67d7 100644 --- a/eval/src/tests/instruction/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp +++ b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp @@ -19,7 +19,7 @@ EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("a", spec(1.5)) .add("b", spec(2.5)) - .add("sparse", spec({x({"a"})}, N())) + .add("sparse", spec({x({"a","b"})}, N())) .add("mixed", spec({x({"a"}),y(5)}, N())) .add_matrix("x", 5, "y", 3); } @@ -30,11 +30,10 @@ void verify_optimized(const vespalib::string &expr, op1_t op1, bool inplace = fa EvalFixture fixture(prod_factory, expr, param_repo, true, true); EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); EXPECT_EQ(fixture.result(), slow_fixture.result()); - auto info = fixture.find_all<DenseSimpleMapFunction>(); + auto info = fixture.find_all<tensor_function::Map>(); ASSERT_EQ(info.size(), 1u); EXPECT_TRUE(info[0]->result_is_mutable()); EXPECT_EQ(info[0]->function(), op1); - EXPECT_EQ(info[0]->inplace(), inplace); ASSERT_EQ(fixture.num_params(), 1); if (inplace) { EXPECT_EQ(fixture.get_param(0), fixture.result()); @@ -76,16 +75,16 @@ TEST(PowAsMapTest, hypercubed_dense_tensor_is_not_optimized) { verify_not_optimized("join(x5y3,4.0,f(x,y)(pow(x,y)))"); } -TEST(PowAsMapTest, scalar_join_is_not_optimized) { - verify_not_optimized("join(a,2.0,f(x,y)(pow(x,y)))"); +TEST(PowAsMapTest, scalar_join_is_optimized) { + verify_optimized("join(a,2.0,f(x,y)(pow(x,y)))", Square::f); } -TEST(PowAsMapTest, sparse_join_is_not_optimized) { - verify_not_optimized("join(sparse,2.0,f(x,y)(pow(x,y)))"); +TEST(PowAsMapTest, sparse_join_is_optimized) { + verify_optimized("join(sparse,2.0,f(x,y)(pow(x,y)))", Square::f); } -TEST(PowAsMapTest, mixed_join_is_not_optimized) { - verify_not_optimized("join(mixed,2.0,f(x,y)(pow(x,y)))"); +TEST(PowAsMapTest, mixed_join_is_optimized) { + verify_optimized("join(mixed,2.0,f(x,y)(pow(x,y)))", Square::f); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 89bddfda933..3345d7dc8ee 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -814,6 +814,14 @@ TEST(DenseJoin, partial_overlap) { benchmark_join("dense partial overlap multiply", lhs, rhs, operation::Mul::f); } +TEST(DenseJoin, subset_overlap) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto rhs_inner = make_matrix(D::idx("b", 16), D::idx("c", 16), 2.0); + auto rhs_outer = make_matrix(D::idx("a", 16), D::idx("b", 16), 3.0); + benchmark_join("dense subset overlap inner multiply", lhs, rhs_inner, operation::Mul::f); + benchmark_join("dense subset overlap outer multiply", lhs, rhs_outer, operation::Mul::f); +} + TEST(DenseJoin, no_overlap) { auto lhs = make_cube(D::idx("a", 4), D::idx("e", 4), D::idx("f", 4), 1.0); auto rhs = make_cube(D::idx("b", 4), D::idx("c", 4), D::idx("d", 4), 2.0); diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index d2e7a64b51b..516ead0f0bf 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -61,9 +61,10 @@ struct Aggregator { namespace aggr { -// can we start by picking any value from the set to be reduced and -// use the templated aggregator 'combine' function in arbitrary order -// to end up with (approximately) the correct result? +// can we start by picking any value from the set to be reduced (or +// the special aggregator-specific null_value) and use the templated +// aggregator 'combine' function in arbitrary order to end up with +// (approximately) the correct result? constexpr bool is_simple(Aggr aggr) { return ((aggr == Aggr::PROD) || (aggr == Aggr::SUM) || @@ -124,12 +125,13 @@ private: T _prod; public: using value_type = T; - constexpr Prod() : _prod{1} {} + constexpr Prod() : _prod{null_value()} {} constexpr Prod(T value) : _prod{value} {} - constexpr void sample(T value) { _prod *= value; } - constexpr void merge(const Prod &rhs) { _prod *= rhs._prod; } + constexpr void sample(T value) { _prod = combine(_prod, value); } + constexpr void merge(const Prod &rhs) { _prod = combine(_prod, rhs._prod); } constexpr T result() const { return _prod; } static constexpr Aggr enum_value() { return Aggr::PROD; } + static constexpr T null_value() { return 1; } static constexpr T combine(T a, T b) { return (a * b); } }; @@ -138,12 +140,13 @@ private: T _sum; public: using value_type = T; - constexpr Sum() : _sum{0} {} + constexpr Sum() : _sum{null_value()} {} constexpr Sum(T value) : _sum{value} {} - constexpr void sample(T value) { _sum += value; } - constexpr void merge(const Sum &rhs) { _sum += rhs._sum; } + constexpr void sample(T value) { _sum = combine(_sum, value); } + constexpr void merge(const Sum &rhs) { _sum = combine(_sum, rhs._sum); } constexpr T result() const { return _sum; } static constexpr Aggr enum_value() { return Aggr::SUM; } + static constexpr T null_value() { return 0; } static constexpr T combine(T a, T b) { return (a + b); } }; @@ -152,12 +155,13 @@ private: T _max; public: using value_type = T; - constexpr Max() : _max{-std::numeric_limits<T>::infinity()} {} + constexpr Max() : _max{null_value()} {} constexpr Max(T value) : _max{value} {} - constexpr void sample(T value) { _max = std::max(_max, value); } - constexpr void merge(const Max &rhs) { _max = std::max(_max, rhs._max); } + constexpr void sample(T value) { _max = combine(_max, value); } + constexpr void merge(const Max &rhs) { _max = combine(_max, rhs._max); } constexpr T result() const { return _max; } static constexpr Aggr enum_value() { return Aggr::MAX; } + static constexpr T null_value() { return -std::numeric_limits<T>::infinity(); } static constexpr T combine(T a, T b) { return std::max(a,b); } }; @@ -204,12 +208,13 @@ private: T _min; public: using value_type = T; - constexpr Min() : _min{std::numeric_limits<T>::infinity()} {} + constexpr Min() : _min{null_value()} {} constexpr Min(T value) : _min{value} {} - constexpr void sample(T value) { _min = std::min(_min, value); } - constexpr void merge(const Min &rhs) { _min = std::min(_min, rhs._min); } + constexpr void sample(T value) { _min = combine(_min, value); } + constexpr void merge(const Min &rhs) { _min = combine(_min, rhs._min); } constexpr T result() const { return _min; } static constexpr Aggr enum_value() { return Aggr::MIN; } + static constexpr T null_value() { return std::numeric_limits<T>::infinity(); } static constexpr T combine(T a, T b) { return std::min(a,b); } }; diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index 515b48b3693..0d7a6937c0d 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -5,6 +5,7 @@ #include "simple_value.h" #include <vespa/eval/instruction/dense_dot_product_function.h> +#include <vespa/eval/instruction/mixed_inner_product_function.h> #include <vespa/eval/instruction/dense_xw_product_function.h> #include <vespa/eval/instruction/dense_matmul_function.h> #include <vespa/eval/instruction/dense_multi_matmul_function.h> @@ -16,7 +17,7 @@ #include <vespa/eval/instruction/dense_simple_expand_function.h> #include <vespa/eval/instruction/dense_simple_join_function.h> #include <vespa/eval/instruction/join_with_number_function.h> -#include <vespa/eval/instruction/dense_pow_as_map_optimizer.h> +#include <vespa/eval/instruction/pow_as_map_optimizer.h> #include <vespa/eval/instruction/dense_simple_map_function.h> #include <vespa/eval/instruction/vector_from_doubles_function.h> #include <vespa/eval/instruction/dense_tensor_create_function.h> @@ -47,6 +48,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &factory, c child.set(DenseXWProductFunction::optimize(child.get(), stash)); child.set(DenseMatMulFunction::optimize(child.get(), stash)); child.set(DenseMultiMatMulFunction::optimize(child.get(), stash)); + child.set(MixedInnerProductFunction::optimize(child.get(), stash)); nodes.pop_back(); } } @@ -65,7 +67,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &factory, c child.set(DenseTensorPeekFunction::optimize(child.get(), stash)); child.set(DenseLambdaPeekOptimizer::optimize(child.get(), stash)); child.set(FastRenameOptimizer::optimize(child.get(), stash)); - child.set(DensePowAsMapOptimizer::optimize(child.get(), stash)); + child.set(PowAsMapOptimizer::optimize(child.get(), stash)); child.set(DenseSimpleMapFunction::optimize(child.get(), stash)); child.set(DenseSimpleJoinFunction::optimize(child.get(), stash)); child.set(JoinWithNumberFunction::optimize(child.get(), stash)); diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index 1317a2c2cf6..4b3b357e2b9 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -9,8 +9,6 @@ vespa_add_library(eval_instruction OBJECT dense_lambda_peek_optimizer.cpp dense_matmul_function.cpp dense_multi_matmul_function.cpp - dense_pow_as_map_optimizer.cpp - remove_trivial_dimension_optimizer.cpp dense_simple_expand_function.cpp dense_simple_join_function.cpp dense_simple_map_function.cpp @@ -30,6 +28,9 @@ vespa_add_library(eval_instruction OBJECT generic_rename.cpp index_lookup_table.cpp join_with_number_function.cpp + mixed_inner_product_function.cpp + pow_as_map_optimizer.cpp + remove_trivial_dimension_optimizer.cpp replace_type_function.cpp vector_from_doubles_function.cpp ) diff --git a/eval/src/vespa/eval/instruction/generic_reduce.cpp b/eval/src/vespa/eval/instruction/generic_reduce.cpp index b6393d0d713..2d4144d64b1 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.cpp +++ b/eval/src/vespa/eval/instruction/generic_reduce.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/util/visit_ranges.h> +#include <algorithm> #include <cassert> #include <array> @@ -48,7 +49,7 @@ struct SparseReduceState { std::vector<string_id> full_address; std::vector<string_id*> fetch_address; std::vector<string_id*> keep_address; - size_t subspace; + size_t subspace; SparseReduceState(const SparseReducePlan &plan) : full_address(plan.keep_dims.size() + plan.num_reduce_dims), @@ -72,8 +73,8 @@ Value::UP generic_reduce(const Value &value, const ReduceParam ¶m) { auto cells = value.cells().typify<ICT>(); ArrayArrayMap<string_id,AGGR> map(param.sparse_plan.keep_dims.size(), - param.dense_plan.out_size, - value.index().size()); + param.dense_plan.out_size, + value.index().size()); SparseReduceState sparse(param.sparse_plan); auto full_view = value.index().create_view({}); full_view->lookup({}); @@ -94,9 +95,7 @@ generic_reduce(const Value &value, const ReduceParam ¶m) { }); if ((map.size() == 0) && param.sparse_plan.keep_dims.empty()) { auto zero = builder->add_subspace(); - for (size_t i = 0; i < zero.size(); ++i) { - zero[i] = OCT{}; - } + std::fill(zero.begin(), zero.end(), OCT{}); } return builder->build(std::move(builder)); } @@ -109,6 +108,50 @@ void my_generic_reduce_op(State &state, uint64_t param_in) { auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(up)); const Value &result_ref = *(result.get()); state.pop_push(result_ref); +} + +template <typename ICT, typename OCT, typename AGGR, bool forward_index> +void my_generic_dense_reduce_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<ReduceParam>(param_in); + const Value &value = state.peek(0); + auto cells = value.cells().typify<ICT>(); + const auto &index = value.index(); + size_t num_subspaces = index.size(); + size_t out_cells_size = forward_index ? (param.dense_plan.out_size * num_subspaces) : param.dense_plan.out_size; + auto out_cells = state.stash.create_uninitialized_array<OCT>(out_cells_size); + if (num_subspaces > 0) { + if constexpr (aggr::is_simple(AGGR::enum_value())) { + OCT *dst = out_cells.begin(); + std::fill(out_cells.begin(), out_cells.end(), AGGR::null_value()); + auto combine = [&](size_t src_idx, size_t dst_idx) { dst[dst_idx] = AGGR::combine(dst[dst_idx], cells[src_idx]); }; + for (size_t i = 0; i < num_subspaces; ++i) { + param.dense_plan.execute(i * param.dense_plan.in_size, combine); + if (forward_index) { + dst += param.dense_plan.out_size; + } + } + } else { + std::vector<AGGR> aggr_state(out_cells_size); + AGGR *dst = &aggr_state[0]; + auto sample = [&](size_t src_idx, size_t dst_idx) { dst[dst_idx].sample(cells[src_idx]); }; + for (size_t i = 0; i < num_subspaces; ++i) { + param.dense_plan.execute(i * param.dense_plan.in_size, sample); + if (forward_index) { + dst += param.dense_plan.out_size; + } + } + for (size_t i = 0; i < aggr_state.size(); ++i) { + out_cells[i] = aggr_state[i].result(); + } + } + } else if (!forward_index) { + std::fill(out_cells.begin(), out_cells.end(), OCT{}); + } + if (forward_index) { + state.pop_push(state.stash.create<ValueView>(param.res_type, index, TypedCells(out_cells))); + } else { + state.pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells))); + } }; template <typename ICT, typename OCT, typename AGGR> @@ -147,10 +190,17 @@ void my_full_reduce_op(State &state, uint64_t) { struct SelectGenericReduceOp { template <typename ICT, typename OCT, typename AGGR> static auto invoke(const ReduceParam ¶m) { + using AggrType = typename AGGR::template templ<OCT>; if (param.res_type.is_scalar()) { - return my_full_reduce_op<ICT, OCT, typename AGGR::template templ<OCT>>; + return my_full_reduce_op<ICT, OCT, AggrType>; + } + if (param.sparse_plan.should_forward_index()) { + return my_generic_dense_reduce_op<ICT, OCT, AggrType, true>; } - return my_generic_reduce_op<ICT, OCT, typename AGGR::template templ<OCT>>; + if (param.res_type.is_dense()) { + return my_generic_dense_reduce_op<ICT, OCT, AggrType, false>; + } + return my_generic_reduce_op<ICT, OCT, AggrType>; } }; @@ -227,6 +277,12 @@ SparseReducePlan::SparseReducePlan(const ValueType &type, const ValueType &res_t } } +bool +SparseReducePlan::should_forward_index() const +{ + return ((num_reduce_dims == 0) && (!keep_dims.empty())); +} + SparseReducePlan::~SparseReducePlan() = default; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/instruction/generic_reduce.h b/eval/src/vespa/eval/instruction/generic_reduce.h index f753a3e51cd..5faafb0325d 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.h +++ b/eval/src/vespa/eval/instruction/generic_reduce.h @@ -30,6 +30,7 @@ struct DenseReducePlan { struct SparseReducePlan { size_t num_reduce_dims; std::vector<size_t> keep_dims; + bool should_forward_index() const; SparseReducePlan(const ValueType &type, const ValueType &res_type); ~SparseReducePlan(); }; diff --git a/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp b/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp new file mode 100644 index 00000000000..c8a4df2b82d --- /dev/null +++ b/eval/src/vespa/eval/instruction/mixed_inner_product_function.cpp @@ -0,0 +1,158 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mixed_inner_product_function.h" +#include <vespa/eval/eval/operation.h> +#include <vespa/eval/eval/value.h> +#include <cblas.h> + +namespace vespalib::eval { + +using namespace tensor_function; +using namespace operation; + +namespace { + +template <typename LCT, typename RCT> +struct MyDotProduct { + static double apply(const LCT * lhs, const RCT * rhs, size_t count) { + double result = 0.0; + for (size_t i = 0; i < count; ++i) { + result += lhs[i] * rhs[i]; + } + return result; + } +}; + +template <> +struct MyDotProduct<double,double> { + static double apply(const double * lhs, const double * rhs, size_t count) { + return cblas_ddot(count, lhs, 1, rhs, 1); + } +}; + +template <> +struct MyDotProduct<float,float> { + static float apply(const float * lhs, const float * rhs, size_t count) { + return cblas_sdot(count, lhs, 1, rhs, 1); + } +}; + +struct MixedInnerProductParam { + ValueType res_type; + size_t vector_size; + size_t out_subspace_size; + + MixedInnerProductParam(const ValueType &res_type_in, + const ValueType &mix_type, + const ValueType &vec_type) + : res_type(res_type_in), + vector_size(vec_type.dense_subspace_size()), + out_subspace_size(res_type.dense_subspace_size()) + { + assert(vector_size * out_subspace_size == mix_type.dense_subspace_size()); + } +}; + +template <typename MCT, typename VCT, typename OCT> +void my_mixed_inner_product_op(InterpretedFunction::State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<MixedInnerProductParam>(param_in); + const auto &mixed = state.peek(1); + const auto &vector = state.peek(0); + auto m_cells = mixed.cells().typify<MCT>(); + auto v_cells = vector.cells().typify<VCT>(); + const auto &index = mixed.index(); + size_t num_subspaces = index.size(); + size_t num_output_cells = num_subspaces * param.out_subspace_size; + ArrayRef<OCT> out_cells = state.stash.create_uninitialized_array<OCT>(num_output_cells); + const MCT *m_cp = m_cells.begin(); + const VCT *v_cp = v_cells.begin(); + for (OCT &out : out_cells) { + out = MyDotProduct<MCT,VCT>::apply(m_cp, v_cp, param.vector_size); + m_cp += param.vector_size; + } + assert(m_cp == m_cells.end()); + state.pop_pop_push(state.stash.create<ValueView>(param.res_type, index, TypedCells(out_cells))); +} + + +struct SelectMixedInnerProduct { + template <typename MCT, typename VCT, typename OCT> + static auto invoke() { return my_mixed_inner_product_op<MCT,VCT,OCT>; } +}; + +} // namespace <unnamed> + +MixedInnerProductFunction::MixedInnerProductFunction(const ValueType &res_type_in, + const TensorFunction &mixed_child, + const TensorFunction &vector_child) + : tensor_function::Op2(res_type_in, mixed_child, vector_child) +{ +} + +InterpretedFunction::Instruction +MixedInnerProductFunction::compile_self(const ValueBuilderFactory &, Stash &stash) const +{ + const auto &mix_type = lhs().result_type(); + const auto &vec_type = rhs().result_type(); + auto ¶m = stash.create<MixedInnerProductParam>(result_type(), mix_type, vec_type); + using MyTypify = TypifyValue<TypifyCellType>; + auto op = typify_invoke<3,MyTypify,SelectMixedInnerProduct>(mix_type.cell_type(), + vec_type.cell_type(), + result_type().cell_type()); + return InterpretedFunction::Instruction(op, wrap_param<MixedInnerProductParam>(param)); +} + +bool +MixedInnerProductFunction::compatible_types(const ValueType &res, const ValueType &mixed, const ValueType &vector) +{ + if (vector.is_dense() && ! res.is_scalar()) { + auto dense_dims = vector.nontrivial_indexed_dimensions(); + auto mixed_dims = mixed.nontrivial_indexed_dimensions(); + while (! dense_dims.empty()) { + if (mixed_dims.empty()) { + return false; + } + const auto &name = dense_dims.back().name; + if (res.dimension_index(name) != ValueType::Dimension::npos) { + return false; + } + if (name != mixed_dims.back().name) { + return false; + } + dense_dims.pop_back(); + mixed_dims.pop_back(); + } + while (! mixed_dims.empty()) { + const auto &name = mixed_dims.back().name; + if (res.dimension_index(name) == ValueType::Dimension::npos) { + return false; + } + mixed_dims.pop_back(); + } + return (res.mapped_dimensions() == mixed.mapped_dimensions()); + } + return false; +} + +const TensorFunction & +MixedInnerProductFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + const auto & res_type = expr.result_type(); + auto reduce = as<Reduce>(expr); + if ((! res_type.is_scalar()) && reduce && (reduce->aggr() == Aggr::SUM)) { + auto join = as<Join>(reduce->child()); + if (join && (join->function() == Mul::f)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (compatible_types(res_type, lhs.result_type(), rhs.result_type())) { + return stash.create<MixedInnerProductFunction>(res_type, lhs, rhs); + } + if (compatible_types(res_type, rhs.result_type(), lhs.result_type())) { + return stash.create<MixedInnerProductFunction>(res_type, rhs, lhs); + } + } + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/mixed_inner_product_function.h b/eval/src/vespa/eval/instruction/mixed_inner_product_function.h new file mode 100644 index 00000000000..99ef88997b6 --- /dev/null +++ b/eval/src/vespa/eval/instruction/mixed_inner_product_function.h @@ -0,0 +1,46 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Tensor function for a dot product inside a mixed tensor. + * + * Optimized tensor function for dot-product inside a bigger (possibly + * mixed) tensor. To trigger this, the function must be in the form + * reduce((mixed tensor)*(vector),sum,dimension names) + * with "vector" being a dense tensor with the same dimensions that + * are reduced, "mixed tensor" must contain all these dimension, and + * they must also be the innermost (alphabetically last) indexed + * dimensions in the mixed tensor. + * Simple example: + * mixed: tensor(category{},x[32]) + * vector: tensor(x[32]) + * expression: reduce(mixed*vector,sum,x) + * result: tensor(category{}) + * More complex example: + * mixed: tensor<double>(a{},b[31],c{},d[42],e{},f[5],g{}) + * vector: tensor<float>(d[42],f[5]) + * expression: reduce(mixed*vector,sum,d,f) + * result: tensor<double>(a{},b[31],c{},e{},g{}) + * Note: + * if the bigger tensor is dense, other optimizers are likely + * to pick up the operation, even if this function could also + * handle them. + **/ +class MixedInnerProductFunction : public tensor_function::Op2 +{ +public: + MixedInnerProductFunction(const ValueType &res_type_in, + const TensorFunction &mixed_child, + const TensorFunction &vector_child); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + static bool compatible_types(const ValueType &res, const ValueType &mixed, const ValueType &dense); + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.cpp b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.cpp index 61ef2243480..00fbc1c642b 100644 --- a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.cpp +++ b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.cpp @@ -1,6 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dense_pow_as_map_optimizer.h" +#include "pow_as_map_optimizer.h" #include "dense_simple_map_function.h" #include <vespa/eval/eval/operation.h> @@ -10,14 +10,13 @@ using namespace tensor_function; using namespace operation; const TensorFunction & -DensePowAsMapOptimizer::optimize(const TensorFunction &expr, Stash &stash) +PowAsMapOptimizer::optimize(const TensorFunction &expr, Stash &stash) { if (auto join = as<Join>(expr)) { const TensorFunction &lhs = join->lhs(); const TensorFunction &rhs = join->rhs(); if ((join->function() == Pow::f) && - lhs.result_type().is_dense() && - rhs.result_type().is_double()) + rhs.result_type().is_scalar()) { if (auto const_value = as<ConstValue>(rhs)) { if (const_value->value().as_double() == 2.0) { diff --git a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.h b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.h index e61069b87b0..df9735f76ca 100644 --- a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.h +++ b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.h @@ -10,9 +10,8 @@ namespace vespalib::eval { * Tensor function optimizer for converting join expressions on the * form 'join(tensor,<small integer constant>,f(x,y)(pow(x,y))' to * expressions on the form 'map(tensor,f(x)(x*x...))'. - * TODO: extend to mixed tensors. **/ -struct DensePowAsMapOptimizer { +struct PowAsMapOptimizer { static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c4805b3651b..6056fdd9a18 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -285,6 +285,18 @@ public class Flags { "Whether to enable jdisc connection log", "Takes effect on (re)deployment"); + public static final UnboundBooleanFlag ENABLE_ZSTD_COMPRESSION_ACCESS_LOG = defineFeatureFlag( + "enable-zstd-compression-access-log", false, + List.of("bjorncs", "tokle", "baldersheim"), "2021-01-19", "2021-04-01", + "Whether to enable zstd compression of jdisc access logs", + "Takes effect on (re)deployment"); + + public static final UnboundBooleanFlag USE_ENDPOINT_CERTIFICATE_MAINTAINER = defineFeatureFlag( + "use-endpoint-certificate-maintainer", false, + List.of("andreer"), "2021-01-12", "2021-02-12", + "Use EndpointCertificateMaintainer instead of EndpointCertificateManager cleanup thread to handle certificate refreshes and deletions", + "Takes effect on next run of maintainer / next manager cleanup thread run"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/functions.cmake b/functions.cmake index bdecc2f5124..0152554669a 100644 --- a/functions.cmake +++ b/functions.cmake @@ -702,7 +702,15 @@ function(vespa_detect_build_platform) file(STRINGS /etc/os-release OS_DISTRO REGEX "^ID=") string(REGEX REPLACE "ID=\"?([^\"]+)\"?" "\\1" OS_DISTRO ${OS_DISTRO}) file(STRINGS /etc/os-release OS_DISTRO_VERSION REGEX "^VERSION_ID=") - string(REGEX REPLACE "VERSION_ID=\"?([^\"]+)\"?" "\\1" OS_DISTRO_VERSION ${OS_DISTRO_VERSION}) + if(OS_DISTRO_VERSION) + string(REGEX REPLACE "VERSION_ID=\"?([^\"]+)\"?" "\\1" OS_DISTRO_VERSION ${OS_DISTRO_VERSION}) + else() + if (OS_DISTRO STREQUAL "debian") + set(OS_DISTRO_VERSION "sid") + else() + message(FATAL_ERROR "-- Could not determine ${OS_DISTRO} version") + endif() + endif() elseif(EXISTS /etc/redhat-release) set(OS_DISTRO "rhel") file(STRINGS "/etc/redhat-release" OS_DISTRO_VERSION) diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogHandler.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogHandler.java index 488a6137cc2..14236452ac0 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLogHandler.java @@ -10,31 +10,28 @@ import java.util.logging.Logger; */ class AccessLogHandler { - public Logger access = Logger.getAnonymousLogger(); - private LogFileHandler logFileHandler; + public final Logger access = Logger.getAnonymousLogger(); + private final LogFileHandler logFileHandler; - public AccessLogHandler(AccessLogConfig.FileHandler config) { + AccessLogHandler(AccessLogConfig.FileHandler config) { access.setUseParentHandlers(false); - logFileHandler = new LogFileHandler(config.compressOnRotation()); - - logFileHandler.setFilePattern(config.pattern()); - logFileHandler.setRotationTimes(config.rotation()); - - createSymlink(config, logFileHandler); - LogFormatter lf = new LogFormatter(); lf.messageOnly(true); - this.logFileHandler.setFormatter(lf); + logFileHandler = new LogFileHandler(toCompression(config), config.pattern(), config.rotation(), config.symlink(), lf); access.addHandler(this.logFileHandler); } - private void createSymlink(AccessLogConfig.FileHandler config, LogFileHandler handler) { - if (!config.symlink().isEmpty()) - handler.setSymlinkName(config.symlink()); + private LogFileHandler.Compression toCompression(AccessLogConfig.FileHandler config) { + if (!config.compressOnRotation()) return LogFileHandler.Compression.NONE; + switch (config.compressionFormat()) { + case ZSTD: return LogFileHandler.Compression.ZSTD; + case GZIP: return LogFileHandler.Compression.GZIP; + default: throw new IllegalArgumentException(config.compressionFormat().toString()); + } } - public void shutdown() { + void shutdown() { logFileHandler.close(); access.removeHandler(logFileHandler); diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java index 1e64ac1618c..c2013aeb128 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java @@ -14,11 +14,14 @@ class ConnectionLogHandler { public ConnectionLogHandler(String clusterName) { connection.setUseParentHandlers(false); - logFileHandler = new LogFileHandler(true); - logFileHandler.setFilePattern(String.format("logs/vespa/qrs/connection.%s.%s", clusterName, "%Y%m%d%H%M%S")); - LogFormatter lf = new LogFormatter(); lf.messageOnly(true); + logFileHandler = new LogFileHandler( + LogFileHandler.Compression.ZSTD, + String.format("logs/vespa/qrs/ConnectionLog.%s.%s", clusterName, "%Y%m%d%H%M%S"), + "0 60 ...", + String.format("ConnectionLog.%s", clusterName), + lf); this.logFileHandler.setFormatter(lf); connection.addHandler(this.logFileHandler); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/FileConnectionLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/FileConnectionLog.java index 16e49730fdd..f7f4e1e8ed3 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/FileConnectionLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/FileConnectionLog.java @@ -24,7 +24,7 @@ public class FileConnectionLog extends AbstractComponent implements ConnectionLo @Override public void log(ConnectionLogEntry connectionLogEntry) { try { - logHandler.connection.log(Level.INFO, connectionLogEntry.toJson()); + logHandler.connection.log(Level.INFO, connectionLogEntry.toJson()+ '\n'); } catch (Exception e) { logger.log(Level.WARNING, "Unable to write connection log entry for connection id " + connectionLogEntry.id(), e); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 75e9febc192..5b30ffbe363 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; +import com.yahoo.compress.ZstdOuputStream; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.io.NativeIO; import com.yahoo.log.LogFileDb; @@ -11,12 +12,17 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.logging.Formatter; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; @@ -24,29 +30,31 @@ import java.util.logging.StreamHandler; import java.util.zip.GZIPOutputStream; /** - * <p>Implements log file naming/rotating logic for container logs.</p> - * - * <p>Overridden methods: publish</p> - * - * <p>Added methods: setFilePattern, setRotationTimes, rotateNow (+ few others)</p> + * Implements log file naming/rotating logic for container logs. * * @author Bob Travis + * @author bjorncs */ -public class LogFileHandler extends StreamHandler { +class LogFileHandler extends StreamHandler { + + enum Compression { NONE, GZIP, ZSTD } private final static Logger logger = Logger.getLogger(LogFileHandler.class.getName()); - private final boolean compressOnRotation; - private long[] rotationTimes = {0}; //default to one log per day, at midnight - private String filePattern = "./log.%T"; // default to current directory, ms time stamp - private long nextRotationTime = 0; - private FileOutputStream currentOutputStream = null; - private volatile String fileName; - private String symlinkName = null; - private ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(100000); - private LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow"); - private ExecutorService executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("logfilehandler.compression")); + + private final Compression compression; + private final long[] rotationTimes; + private final String filePattern; // default to current directory, ms time stamp + private final String symlinkName; + private final ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(100000); + private final LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow"); + private final ExecutorService executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("logfilehandler.compression")); private final NativeIO nativeIO = new NativeIO(); - private long lastDropPosition = 0; + private final LogThread logThread; + + private volatile FileOutputStream currentOutputStream = null; + private volatile long nextRotationTime = 0; + private volatile String fileName; + private volatile long lastDropPosition = 0; static private class LogThread extends Thread { LogFileHandler logFileHandler; @@ -93,18 +101,25 @@ public class LogFileHandler extends StreamHandler { } } } - private final LogThread logThread; - LogFileHandler() { - this(false); + LogFileHandler(Compression compression, String filePattern, String rotationTimes, String symlinkName, Formatter formatter) { + this(compression, filePattern, calcTimesMinutes(rotationTimes), symlinkName, formatter); } - LogFileHandler(boolean compressOnRotation) - { + LogFileHandler( + Compression compression, + String filePattern, + long[] rotationTimes, + String symlinkName, + Formatter formatter) { super(); - this.compressOnRotation = compressOnRotation; - logThread = new LogThread(this); - logThread.start(); + super.setFormatter(formatter); + this.compression = compression; + this.filePattern = filePattern; + this.rotationTimes = rotationTimes; + this.symlinkName = (symlinkName != null && !symlinkName.isBlank()) ? symlinkName : null; + this.logThread = new LogThread(this); + this.logThread.start(); } /** @@ -112,6 +127,7 @@ public class LogFileHandler extends StreamHandler { * * @param r logrecord to publish */ + @Override public void publish(LogRecord r) { try { logQueue.put(r); @@ -123,7 +139,7 @@ public class LogFileHandler extends StreamHandler { public synchronized void flush() { super.flush(); try { - if (currentOutputStream != null) { + if (currentOutputStream != null && compression == Compression.GZIP) { long newPos = currentOutputStream.getChannel().position(); if (newPos > lastDropPosition + 102400) { nativeIO.dropPartialFileFromCache(currentOutputStream.getFD(), lastDropPosition, newPos, true); @@ -150,33 +166,6 @@ public class LogFileHandler extends StreamHandler { } /** - * Assign pattern for generating (rotating) file names. - * - * @param pattern See LogFormatter for definition - */ - void setFilePattern ( String pattern ) { - filePattern = pattern; - } - - /** - * Assign times for rotating output files. - * - * @param timesOfDay in millis, from midnight - * - */ - void setRotationTimes ( long[] timesOfDay ) { - rotationTimes = timesOfDay; - } - - /** Assign time for rotating output files - * - * @param prescription string form of times, in minutes - */ - void setRotationTimes ( String prescription ) { - setRotationTimes(calcTimesMinutes(prescription)); - } - - /** * Find next rotation after specified time. * * @param now the specified time; if zero, current time is used. @@ -259,8 +248,8 @@ public class LogFileHandler extends StreamHandler { if ((oldFileName != null)) { File oldFile = new File(oldFileName); if (oldFile.exists()) { - if (compressOnRotation) { - executor.execute(() -> runCompression(oldFile)); + if (compression != Compression.NONE) { + executor.execute(() -> runCompression(oldFile, compression)); } else { nativeIO.dropFileFromCache(oldFile); } @@ -269,7 +258,40 @@ public class LogFileHandler extends StreamHandler { } - static void runCompression(File oldFile) { + private static void runCompression(File oldFile, Compression compression) { + switch (compression) { + case ZSTD: + runCompressionZstd(oldFile.toPath()); + break; + case GZIP: + runCompressionGzip(oldFile); + break; + default: + throw new IllegalArgumentException("Unknown compression " + compression); + } + } + + private static void runCompressionZstd(Path oldFile) { + try { + Path compressedFile = Paths.get(oldFile.toString() + ".zst"); + Files.createFile(compressedFile); + int bufferSize = 0x400000; // 4M + byte[] buffer = new byte[bufferSize]; + try (ZstdOuputStream out = new ZstdOuputStream(Files.newOutputStream(compressedFile), bufferSize); + InputStream in = Files.newInputStream(oldFile)) { + int read; + while ((read = in.read(buffer)) >= 0) { + out.write(buffer, 0, read); + } + out.flush(); + } + Files.delete(oldFile); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to compress log file with zstd: " + oldFile, e); + } + } + + private static void runCompressionGzip(File oldFile) { File gzippedFile = new File(oldFile.getPath() + ".gz"); try (GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000); FileInputStream inputStream = new FileInputStream(oldFile)) @@ -368,15 +390,11 @@ public class LogFileHandler extends StreamHandler { return time % lengthOfDayMillis; } - void setSymlinkName(String symlinkName) { - this.symlinkName = symlinkName; - } - /** * Flushes all queued messages, interrupts the log thread in this and * waits for it to end before returning */ - public void shutdown() { + void shutdown() { logThread.interrupt(); try { logThread.join(); @@ -390,7 +408,7 @@ public class LogFileHandler extends StreamHandler { /** * Only for unit testing. Do not use. */ - public String getFileName() { + String getFileName() { return fileName; } diff --git a/jdisc_http_service/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java b/jdisc_http_service/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java deleted file mode 100644 index 2099037203f..00000000000 --- a/jdisc_http_service/src/test/java/com/yahoo/container/logging/CompressWhileDrop.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.logging; - -import java.io.File; - -public class CompressWhileDrop { - public static void main(String [] args) { - System.out.println("Start compressing file " + args[0]); - LogFileHandler.runCompression(new File(args[0])); - } -} diff --git a/jdisc_http_service/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java index bc7257b1ca9..c0dafe05bb6 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java @@ -1,26 +1,29 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.logging; +import com.yahoo.compress.ZstdCompressor; +import com.yahoo.container.logging.LogFileHandler.Compression; import com.yahoo.io.IOUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.function.BiFunction; import java.util.logging.Formatter; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.SimpleFormatter; import java.util.zip.GZIPInputStream; +import static com.yahoo.yolean.Exceptions.uncheck; import static org.assertj.core.api.Assertions.assertThat; /** @@ -36,22 +39,20 @@ public class LogFileHandlerTestCase { public void testIt() throws IOException { File root = temporaryFolder.newFolder("logfilehandlertest"); - LogFileHandler h = new LogFileHandler(); - h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S"); - h.setFormatter(new Formatter() { - public String format(LogRecord r) { - DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); - String timeStamp = df.format(new Date(r.getMillis())); - return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); - } - } ); + String pattern = root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S"; + long[] rTimes = {1000, 2000, 10000}; + Formatter formatter = new Formatter() { + public String format(LogRecord r) { + DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); + String timeStamp = df.format(new Date(r.getMillis())); + return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); + } + }; + LogFileHandler h = new LogFileHandler(Compression.NONE, pattern, rTimes, null, formatter); long now = System.currentTimeMillis(); long millisPerDay = 60*60*24*1000; long tomorrowDays = (now / millisPerDay) +1; long tomorrowMillis = tomorrowDays * millisPerDay; - assertThat(tomorrowMillis).isEqualTo(h.getNextRotationTime(now)); - long[] rTimes = {1000, 2000, 10000}; - h.setRotationTimes(rTimes); assertThat(tomorrowMillis+1000).isEqualTo(h.getNextRotationTime(tomorrowMillis)); assertThat(tomorrowMillis+10000).isEqualTo(h.getNextRotationTime(tomorrowMillis+3000)); LogRecord lr = new LogRecord(Level.INFO, "test"); @@ -68,10 +69,7 @@ public class LogFileHandlerTestCase { File logFile = temporaryFolder.newFile("testLogFileG1.txt"); //create logfilehandler - LogFileHandler h = new LogFileHandler(); - h.setFilePattern(logFile.getAbsolutePath()); - h.setFormatter(new SimpleFormatter()); - h.setRotationTimes("0 5 ..."); + LogFileHandler h = new LogFileHandler(Compression.NONE, logFile.getAbsolutePath(), "0 5 ...", null, new SimpleFormatter()); //write log LogRecord lr = new LogRecord(Level.INFO, "testDeleteFileFirst1"); @@ -85,10 +83,7 @@ public class LogFileHandlerTestCase { File logFile = temporaryFolder.newFile("testLogFileG2.txt"); //create logfilehandler - LogFileHandler h = new LogFileHandler(); - h.setFilePattern(logFile.getAbsolutePath()); - h.setFormatter(new SimpleFormatter()); - h.setRotationTimes("0 5 ..."); + LogFileHandler h = new LogFileHandler(Compression.NONE, logFile.getAbsolutePath(), "0 5 ...", null, new SimpleFormatter()); //write log LogRecord lr = new LogRecord(Level.INFO, "testDeleteFileDuringLogging1"); @@ -108,16 +103,15 @@ public class LogFileHandlerTestCase { @Test(timeout = /*5 minutes*/300_000) public void testSymlink() throws IOException, InterruptedException { File root = temporaryFolder.newFolder("testlogforsymlinkchecking"); - LogFileHandler handler = new LogFileHandler(); - handler.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); - handler.setFormatter(new Formatter() { + Formatter formatter = new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); String timeStamp = df.format(new Date(r.getMillis())); - return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); + return ("[" + timeStamp + "]" + " " + formatMessage(r) + "\n"); } - } ); - handler.setSymlinkName("symlink"); + }; + LogFileHandler handler = new LogFileHandler( + Compression.NONE, root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", formatter); handler.publish(new LogRecord(Level.INFO, "test")); String firstFile; @@ -148,18 +142,39 @@ public class LogFileHandlerTestCase { } @Test - public void testcompression() throws InterruptedException, IOException { - File root = temporaryFolder.newFolder("testcompression"); + public void testcompression_gzip() throws InterruptedException, IOException { + testcompression( + Compression.GZIP, "gz", + (compressedFile, __) -> uncheck(() -> new String(new GZIPInputStream(Files.newInputStream(compressedFile)).readAllBytes()))); + } - LogFileHandler h = new LogFileHandler(true); - h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s"); - h.setFormatter(new Formatter() { + @Test + public void testcompression_zstd() throws InterruptedException, IOException { + testcompression( + Compression.ZSTD, "zst", + (compressedFile, uncompressedSize) -> uncheck(() -> { + ZstdCompressor zstdCompressor = new ZstdCompressor(); + byte[] uncompressedBytes = new byte[uncompressedSize]; + byte[] compressedBytes = Files.readAllBytes(compressedFile); + zstdCompressor.decompress(compressedBytes, 0, compressedBytes.length, uncompressedBytes, 0, uncompressedBytes.length); + return new String(uncompressedBytes); + })); + } + + private void testcompression(Compression compression, + String fileExtension, + BiFunction<Path, Integer, String> decompressor) throws IOException, InterruptedException { + File root = temporaryFolder.newFolder("testcompression" + compression.name()); + + Formatter formatter = new Formatter() { public String format(LogRecord r) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS"); String timeStamp = df.format(new Date(r.getMillis())); - return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n"); + return ("[" + timeStamp + "]" + " " + formatMessage(r) + "\n"); } - } ); + }; + LogFileHandler h = new LogFileHandler( + compression, root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s", new long[]{0}, null, formatter); int logEntries = 10000; for (int i = 0; i < logEntries; i++) { LogRecord lr = new LogRecord(Level.INFO, "test"); @@ -169,7 +184,7 @@ public class LogFileHandlerTestCase { String f1 = h.getFileName(); assertThat(f1).startsWith(root.getAbsolutePath() + "/logfilehandlertest."); File uncompressed = new File(f1); - File compressed = new File(f1 + ".gz"); + File compressed = new File(f1 + "." + fileExtension); assertThat(uncompressed).exists(); assertThat(compressed).doesNotExist(); String content = IOUtils.readFile(uncompressed); @@ -179,8 +194,8 @@ public class LogFileHandlerTestCase { Thread.sleep(1); } assertThat(compressed).exists(); - String unzipped = IOUtils.readAll(new InputStreamReader(new GZIPInputStream(new FileInputStream(compressed)))); - assertThat(content).isEqualTo(unzipped); + String uncompressedContent = decompressor.apply(compressed.toPath(), content.getBytes().length); + assertThat(uncompressedContent).isEqualTo(content); h.shutdown(); } diff --git a/node-admin/pom.xml b/node-admin/pom.xml index d636e2544c2..52873501744 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -143,6 +143,12 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/node-repository/pom.xml b/node-repository/pom.xml index beb6821290e..fb46735ec73 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -130,6 +130,12 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 6477b9b1cd0..b16859fa6fb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import java.time.Instant; import java.util.ArrayList; @@ -73,10 +74,18 @@ public class Cluster { /** * The suggested size of this cluster, which may or may not be within the min and max limits, - * or empty if there is currently no suggestion. + * or empty if there is currently no recorded suggestion. */ public Optional<Suggestion> suggestedResources() { return suggested; } + /** Returns true if there is a current suggestion and we should actually make this suggestion to users. */ + public boolean shouldSuggestResources(ClusterResources currentResources) { + if (suggested.isEmpty()) return false; + if (suggested.get().resources().isWithin(min, max)) return false; + if (Autoscaler.similar(suggested.get().resources(), currentResources)) return false; + return true; + } + /** Returns the recent scaling events in this cluster */ public List<ScalingEvent> scalingEvents() { return scalingEvents; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 8f28e307c5d..ab8b954610a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * @author bratseth @@ -69,15 +68,15 @@ public class AllocatableClusterResources { * Returns the resources which will actually be available per node in this cluster with this allocation. * These should be used for reasoning about allocation to meet measured demand. */ - public NodeResources realResources() { return realResources; } + public ClusterResources realResources() { + return new ClusterResources(nodes, groups, realResources); + } /** * Returns the resources advertised by the cloud provider, which are the basis for charging * and which must be used in resource allocation requests */ - public NodeResources advertisedResources() { return advertisedResources; } - - public ClusterResources toAdvertisedClusterResources() { + public ClusterResources advertisedResources() { return new ClusterResources(nodes, groups, advertisedResources); } @@ -115,9 +114,7 @@ public class AllocatableClusterResources { @Override public String toString() { - return nodes + " nodes " + - ( groups > 1 ? "(in " + groups + " groups) " : "" ) + - "with " + advertisedResources() + + return advertisedResources() + " at cost $" + cost() + (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : ""); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 1ec2b841790..bfb3bfeb480 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -98,7 +98,7 @@ public class AllocationOptimizer { // Combine the scaled resource values computed here // with the currently configured non-scaled values, given in the limits, if any NodeResources nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() - ? current.toAdvertisedClusterResources().nodeResources() + ? current.advertisedResources().nodeResources() : limits.min().nodeResources(); // min=max for non-scaled return nonScaled.withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 445f3f7746c..c4ff86a5390 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -94,34 +94,31 @@ public class Autoscaler { if (bestAllocation.isEmpty()) return Advice.dontScale("No allocation changes are possible within configured limits"); - if (similar(bestAllocation.get(), currentAllocation)) + if (similar(bestAllocation.get().realResources(), currentAllocation.realResources())) return Advice.dontScale("Cluster is ideally scaled within configured limits"); if (isDownscaling(bestAllocation.get(), currentAllocation) && scaledIn(scalingWindow.multipliedBy(3), cluster)) return Advice.dontScale("Waiting " + scalingWindow.multipliedBy(3) + " since last rescaling before reducing resources"); - return Advice.scaleTo(bestAllocation.get().toAdvertisedClusterResources()); + return Advice.scaleTo(bestAllocation.get().advertisedResources()); } /** Returns true if both total real resources and total cost are similar */ - private boolean similar(AllocatableClusterResources a, AllocatableClusterResources b) { + public static boolean similar(ClusterResources a, ClusterResources b) { return similar(a.cost(), b.cost(), costDifferenceWorthReallocation) && - similar(a.realResources().vcpu() * a.nodes(), - b.realResources().vcpu() * b.nodes(), resourceDifferenceWorthReallocation) && - similar(a.realResources().memoryGb() * a.nodes(), - b.realResources().memoryGb() * b.nodes(), resourceDifferenceWorthReallocation) && - similar(a.realResources().diskGb() * a.nodes(), - b.realResources().diskGb() * b.nodes(), resourceDifferenceWorthReallocation); + similar(a.totalResources().vcpu(), b.totalResources().vcpu(), resourceDifferenceWorthReallocation) && + similar(a.totalResources().memoryGb(), b.totalResources().memoryGb(), resourceDifferenceWorthReallocation) && + similar(a.totalResources().diskGb(), b.totalResources().diskGb(), resourceDifferenceWorthReallocation); } - private boolean similar(double r1, double r2, double threshold) { + private static boolean similar(double r1, double r2, double threshold) { return Math.abs(r1 - r2) / (( r1 + r2) / 2) < threshold; } /** Returns true if this reduces total resources in any dimension */ private boolean isDownscaling(AllocatableClusterResources target, AllocatableClusterResources current) { - NodeResources targetTotal = target.toAdvertisedClusterResources().totalResources(); - NodeResources currentTotal = current.toAdvertisedClusterResources().totalResources(); + NodeResources targetTotal = target.advertisedResources().totalResources(); + NodeResources currentTotal = current.advertisedResources().totalResources(); return ! targetTotal.justNumbers().satisfies(currentTotal.justNumbers()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java index b6d75165340..ddfb4c48e84 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java @@ -12,24 +12,24 @@ public enum Resource { /** Cpu utilization ratio */ cpu { - double idealAverageLoad() { return 0.2; } + public double idealAverageLoad() { return 0.2; } double valueFrom(NodeResources resources) { return resources.vcpu(); } }, /** Memory utilization ratio */ memory { - double idealAverageLoad() { return 0.7; } + public double idealAverageLoad() { return 0.7; } double valueFrom(NodeResources resources) { return resources.memoryGb(); } }, /** Disk utilization ratio */ disk { - double idealAverageLoad() { return 0.6; } + public double idealAverageLoad() { return 0.6; } double valueFrom(NodeResources resources) { return resources.diskGb(); } }; /** The load we should have of this resource on average, when one node in the cluster is down */ - abstract double idealAverageLoad(); + public abstract double idealAverageLoad(); abstract double valueFrom(NodeResources resources); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java index bf5e53d823b..b00323818d5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceTarget.java @@ -41,7 +41,7 @@ public class ResourceTarget { } private static double nodeUsage(Resource resource, double load, AllocatableClusterResources current) { - return load * resource.valueFrom(current.realResources()); + return load * resource.valueFrom(current.realResources().nodeResources()); } /** Create a target of achieving ideal load given a current load */ @@ -55,9 +55,9 @@ public class ResourceTarget { /** Crete a target of preserving a current allocation */ public static ResourceTarget preserve(AllocatableClusterResources current) { - return new ResourceTarget(current.realResources().vcpu(), - current.realResources().memoryGb(), - current.realResources().diskGb(), + return new ResourceTarget(current.realResources().nodeResources().vcpu(), + current.realResources().nodeResources().memoryGb(), + current.realResources().nodeResources().diskGb(), false); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index c4744f6cb6a..8eb92217356 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.hosted.provision.autoscale.MetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries; import com.yahoo.vespa.hosted.provision.node.History; -import com.yahoo.vespa.orchestrator.status.ApplicationLock; import java.time.Duration; import java.time.Instant; @@ -126,7 +125,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { ApplicationId application, Cluster cluster, NodeList clusterNodes) { - ClusterResources current = new AllocatableClusterResources(clusterNodes.asList(), nodeRepository(), cluster.exclusive()).toAdvertisedClusterResources(); + ClusterResources current = new AllocatableClusterResources(clusterNodes.asList(), nodeRepository(), cluster.exclusive()).advertisedResources(); log.info("Autoscaling " + application + " " + clusterNodes.clusterSpec() + ":" + "\nfrom " + toString(current) + "\nto " + toString(target)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index be8966068c9..3bbebd7798d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -70,7 +70,9 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { if (suggestion.isEmpty()) return false; // Wait only a short time for the lock to avoid interfering with change deployments try (Mutex lock = nodeRepository().lock(applicationId, Duration.ofSeconds(1))) { - applications().get(applicationId).ifPresent(a -> updateSuggestion(suggestion.target(), clusterId, a, lock)); + // empty suggested resources == keep the current allocation, so we record that + var suggestedResources = suggestion.target().orElse(clusterNodes.not().retired().toResources()); + applications().get(applicationId).ifPresent(a -> updateSuggestion(suggestedResources, clusterId, a, lock)); return true; } catch (ApplicationLockException e) { @@ -78,7 +80,7 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { } } - private void updateSuggestion(Optional<ClusterResources> suggestion, + private void updateSuggestion(ClusterResources suggestion, ClusterSpec.Id clusterId, Application application, Mutex lock) { @@ -88,8 +90,8 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { var currentSuggestion = cluster.get().suggestedResources(); if (currentSuggestion.isEmpty() || currentSuggestion.get().at().isBefore(at.minus(Duration.ofDays(7))) - || suggestion.isPresent() && isHigher(suggestion.get(), currentSuggestion.get().resources())) - applications().put(application.with(cluster.get().withSuggested(suggestion.map(s -> new Cluster.Suggestion(s, at)))), lock); + || isHigher(suggestion, currentSuggestion.get().resources())) + applications().put(application.with(cluster.get().withSuggested(Optional.of(new Cluster.Suggestion(suggestion, at)))), lock); } private boolean isHigher(ClusterResources r1, ClusterResources r2) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 33f011f65e2..96bce8b71d4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -179,13 +179,13 @@ public class NodeRepositoryProvisioner implements Provisioner { if (limits.min().equals(limits.max())) return limits.min(); // Don't change current deployments that are still legal - var currentAsAdvertised = current.toAdvertisedClusterResources(); + var currentAsAdvertised = current.advertisedResources(); if (! firstDeployment && currentAsAdvertised.isWithin(limits.min(), limits.max())) return currentAsAdvertised; // Otherwise, find an allocation that preserves the current resources as well as possible return allocationOptimizer.findBestAllocation(ResourceTarget.preserve(current), current, limits) .orElseThrow(() -> new IllegalArgumentException("No allocation possible within " + limits)) - .toAdvertisedClusterResources(); + .advertisedResources(); } private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java index 0530e0cc9b6..f2be6308fd3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -40,17 +40,16 @@ public class ApplicationSerializer { } private static void toSlime(Cluster cluster, List<Node> applicationNodes, Cursor clustersObject) { - List<Node> nodes = NodeList.copyOf(applicationNodes).not().retired().cluster(cluster.id()).asList(); + NodeList nodes = NodeList.copyOf(applicationNodes).not().retired().cluster(cluster.id()); if (nodes.isEmpty()) return; - - int groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); - ClusterResources currentResources = new ClusterResources(nodes.size(), groups, nodes.get(0).resources()); + ClusterResources currentResources = nodes.toResources(); Cursor clusterObject = clustersObject.setObject(cluster.id().value()); toSlime(cluster.minResources(), clusterObject.setObject("min")); toSlime(cluster.maxResources(), clusterObject.setObject("max")); toSlime(currentResources, clusterObject.setObject("current")); - cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested.resources(), clusterObject.setObject("suggested"))); + if (cluster.shouldSuggestResources(currentResources)) + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested.resources(), clusterObject.setObject("suggested"))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray("scalingEvents")); clusterObject.setString("autoscalingStatus", cluster.autoscalingStatus()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 5ec29cb9043..aac9122f8ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -15,16 +15,21 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.MetricSnapshot; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; +import com.yahoo.vespa.hosted.provision.autoscale.Resource; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.time.Duration; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests the scaling suggestions maintainer integration. @@ -67,9 +72,9 @@ public class ScalingSuggestionsMaintainerTest { maintainer.maintain(); assertEquals("14 nodes with [vcpu: 6.9, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", - tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().resources().toString()); + suggestionOf(app1, cluster1, tester).get().resources().toString()); assertEquals("8 nodes with [vcpu: 14.7, memory: 4.0 Gb, disk 11.8 Gb, bandwidth: 0.1 Gbps, storage type: remote]", - tester.nodeRepository().applications().get(app2).get().cluster(cluster2.id()).get().suggestedResources().get().resources().toString()); + suggestionOf(app2, cluster2, tester).get().resources().toString()); // Utilization goes way down tester.clock().advance(Duration.ofHours(13)); @@ -77,14 +82,41 @@ public class ScalingSuggestionsMaintainerTest { maintainer.maintain(); assertEquals("Suggestion stays at the peak value observed", "14 nodes with [vcpu: 6.9, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", - tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().resources().toString()); + suggestionOf(app1, cluster1, tester).get().resources().toString()); // Utilization is still way down and a week has passed tester.clock().advance(Duration.ofDays(7)); addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository(), metricsDb); maintainer.maintain(); assertEquals("Peak suggestion has been outdated", "6 nodes with [vcpu: 2.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, storage type: remote]", - tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().resources().toString()); + suggestionOf(app1, cluster1, tester).get().resources().toString()); + assertTrue(shouldSuggest(app1, cluster1, tester)); + + tester.clock().advance(Duration.ofDays(3)); + addMeasurements(0.7f, 0.7f, 0.7f, 0, 500, app1, tester.nodeRepository(), metricsDb); + maintainer.maintain(); + var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().resources(); + tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, false, true)); + tester.clock().advance(Duration.ofDays(2)); + addMeasurements((float)Resource.cpu.idealAverageLoad(), + (float)Resource.memory.idealAverageLoad(), + (float)Resource.disk.idealAverageLoad(), + 0, 500, app1, tester.nodeRepository(), metricsDb); + maintainer.maintain(); + assertEquals("Suggestion is to keep the current allocation", + suggested, + suggestionOf(app1, cluster1, tester).get().resources()); + assertFalse("Suggestion is not made as it matches what we have", shouldSuggest(app1, cluster1, tester)); + } + + private Optional<Cluster.Suggestion> suggestionOf(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) { + return tester.nodeRepository().applications().get(app).get().cluster(cluster.id()).get().suggestedResources(); + } + + private boolean shouldSuggest(ApplicationId app, ClusterSpec cluster, ProvisioningTester tester) { + var currentResources = tester.nodeRepository().list(app).cluster(cluster.id()).not().retired().toResources(); + return tester.nodeRepository().applications().get(app).get().cluster(cluster.id()).get() + .shouldSuggestResources(currentResources); } public void addMeasurements(float cpu, float memory, float disk, int generation, int count, ApplicationId applicationId, diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 0c1c5db69d6..0865500d3c0 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -8,6 +8,7 @@ #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/persistence/spi/i_resource_usage_listener.h> #include <vespa/persistence/spi/resource_usage.h> +#include <vespa/persistence/spi/bucketexecutor.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -863,6 +864,30 @@ DummyPersistence::register_resource_usage_listener(IResourceUsageListener &liste return {}; } +namespace { + +class SyncExecutorOnDestruction : public vespalib::IDestructorCallback { +public: + explicit SyncExecutorOnDestruction(std::shared_ptr<BucketExecutor> executor) : _executor(std::move(executor)) { } + ~SyncExecutorOnDestruction() override { + if (_executor) { + _executor->sync(); + } + } +private: + std::shared_ptr<BucketExecutor> _executor; +}; + +} + +std::unique_ptr<vespalib::IDestructorCallback> +DummyPersistence::register_executor(std::shared_ptr<BucketExecutor> executor) +{ + assert(_bucket_executor.expired()); + _bucket_executor = executor; + return std::make_unique<SyncExecutorOnDestruction>(executor); +} + std::string DummyPersistence::dumpBucket(const Bucket& b) const { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index c37af0d33eb..ad50648abaf 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -176,6 +176,8 @@ public: Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; std::unique_ptr<vespalib::IDestructorCallback> register_resource_usage_listener(IResourceUsageListener& listener) override; + std::unique_ptr<vespalib::IDestructorCallback> register_executor(std::shared_ptr<BucketExecutor>) override; + std::shared_ptr<BucketExecutor> get_bucket_executor() noexcept { return _bucket_executor.lock(); } /** * The following methods are used only for unit testing. @@ -213,6 +215,7 @@ private: std::condition_variable _cond; std::unique_ptr<ClusterState> _clusterState; + std::weak_ptr<BucketExecutor> _bucket_executor; std::unique_ptr<document::select::Node> parseDocumentSelection( const string& documentSelection, diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt index aad99e12a69..ef871c2584f 100644 --- a/persistence/src/vespa/persistence/spi/CMakeLists.txt +++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(persistence_spi OBJECT exceptions.cpp persistenceprovider.cpp read_consistency.cpp + resource_usage.cpp resource_usage_listener.cpp result.cpp selection.cpp diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index facdb2cadfa..1d873e9a20e 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -30,10 +30,4 @@ AbstractPersistenceProvider::getModifiedBuckets(BucketSpace) const return BucketIdListResult(list); } -std::unique_ptr<vespalib::IDestructorCallback> -AbstractPersistenceProvider::register_executor(std::shared_ptr<BucketExecutor>) -{ - return {}; -} - } diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 5023febe9a2..016928ab10e 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -48,8 +48,6 @@ public: * Default impl empty. */ BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; - - std::unique_ptr<vespalib::IDestructorCallback> register_executor(std::shared_ptr<BucketExecutor>) override; }; } diff --git a/persistence/src/vespa/persistence/spi/bucket_tasks.h b/persistence/src/vespa/persistence/spi/bucket_tasks.h new file mode 100644 index 00000000000..0b9c283817d --- /dev/null +++ b/persistence/src/vespa/persistence/spi/bucket_tasks.h @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "bucketexecutor.h" + +namespace storage::spi { + +/** + * Simple Bucket task that wraps a lambda that does the job. + */ +template<class FunctionType> +class LambdaBucketTask : public BucketTask { +public: + explicit LambdaBucketTask(FunctionType &&func) + : _func(std::move(func)) + {} + + ~LambdaBucketTask() override = default; + + void run(const Bucket & bucket, std::shared_ptr<vespalib::IDestructorCallback> onComplete) override { + _func(bucket, std::move(onComplete)); + } + +private: + FunctionType _func; +}; + +template<class FunctionType> +std::unique_ptr<BucketTask> +makeBucketTask(FunctionType &&function) { + return std::make_unique<LambdaBucketTask<std::decay_t<FunctionType>>> + (std::forward<FunctionType>(function)); +} + +} diff --git a/persistence/src/vespa/persistence/spi/bucketexecutor.h b/persistence/src/vespa/persistence/spi/bucketexecutor.h index 07c86fd5ffb..8237b78cca0 100644 --- a/persistence/src/vespa/persistence/spi/bucketexecutor.h +++ b/persistence/src/vespa/persistence/spi/bucketexecutor.h @@ -3,7 +3,8 @@ #pragma once #include "bucket.h" -#include "operationcomplete.h" + +namespace vespalib { class IDestructorCallback; } namespace storage::spi { @@ -17,7 +18,7 @@ namespace storage::spi { class BucketTask { public: virtual ~BucketTask() = default; - virtual void run(const Bucket & bucket, std::unique_ptr<OperationComplete> onComplete) = 0; + virtual void run(const Bucket & bucket, std::shared_ptr<vespalib::IDestructorCallback> onComplete) = 0; }; /** diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 3237dede24a..81a9484e98c 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -15,7 +15,7 @@ namespace vespalib { class IDestructorCallback; } namespace storage::spi { class IResourceUsageListener; -class BucketExecutor; +struct BucketExecutor; /** * This interface is the basis for a persistence provider in Vespa. A diff --git a/persistence/src/vespa/persistence/spi/resource_usage.cpp b/persistence/src/vespa/persistence/spi/resource_usage.cpp new file mode 100644 index 00000000000..77ef19ea006 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/resource_usage.cpp @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "resource_usage.h" +#include <iostream> + +namespace storage::spi { + +std::ostream& operator<<(std::ostream& out, const ResourceUsage& resource_usage) +{ + out << "{disk_usage=" << resource_usage.get_disk_usage() << + ", memory_usage=" << resource_usage.get_memory_usage() << "}"; + return out; +} + +} diff --git a/persistence/src/vespa/persistence/spi/resource_usage.h b/persistence/src/vespa/persistence/spi/resource_usage.h index 1b2504b0f13..defdaab2fba 100644 --- a/persistence/src/vespa/persistence/spi/resource_usage.h +++ b/persistence/src/vespa/persistence/spi/resource_usage.h @@ -2,6 +2,8 @@ #pragma once +#include <iosfwd> + namespace storage::spi { /* @@ -27,7 +29,14 @@ public: double get_disk_usage() const noexcept { return _disk_usage; } double get_memory_usage() const noexcept { return _memory_usage; } + + bool operator==(const ResourceUsage &rhs) const noexcept { + return (_disk_usage == rhs._disk_usage) && + (_memory_usage == rhs._memory_usage); + } }; +std::ostream& operator<<(std::ostream& out, const ResourceUsage& resource_usage); + } diff --git a/persistence/src/vespa/persistence/spi/resource_usage_listener.cpp b/persistence/src/vespa/persistence/spi/resource_usage_listener.cpp index 123aac2a62b..3ad3fa7182c 100644 --- a/persistence/src/vespa/persistence/spi/resource_usage_listener.cpp +++ b/persistence/src/vespa/persistence/spi/resource_usage_listener.cpp @@ -14,7 +14,7 @@ ResourceUsageListener::ResourceUsageListener() ResourceUsageListener::~ResourceUsageListener() { - _register_guard.reset(); + reset(); } void @@ -29,4 +29,10 @@ ResourceUsageListener::set_register_guard(std::unique_ptr<vespalib::IDestructorC _register_guard = std::move(register_guard); } +void +ResourceUsageListener::reset() +{ + _register_guard.reset(); +} + } diff --git a/persistence/src/vespa/persistence/spi/resource_usage_listener.h b/persistence/src/vespa/persistence/spi/resource_usage_listener.h index ff16c4a011a..d25bc03ed01 100644 --- a/persistence/src/vespa/persistence/spi/resource_usage_listener.h +++ b/persistence/src/vespa/persistence/spi/resource_usage_listener.h @@ -22,6 +22,7 @@ public: void update_resource_usage(const ResourceUsage& resource_usage) override; const ResourceUsage& get_usage() const noexcept { return _usage; } void set_register_guard(std::unique_ptr<vespalib::IDestructorCallback> register_guard); + void reset(); }; } diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp index 73fac7d9439..7f7c0302926 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp @@ -20,10 +20,10 @@ typedef std::vector<uint32_t> LidVector; struct Fixture { DocumentMetaStore _metaStore; - DocumentScanIterator _itr; + std::unique_ptr<DocumentScanIterator> _itr; Fixture() : _metaStore(std::make_shared<BucketDBOwner>()), - _itr(_metaStore) + _itr() { _metaStore.constructFreeList(); } @@ -42,18 +42,25 @@ struct Fixture _metaStore.put(gid, gid.convertToBucketId(), Timestamp(lid), docSize, lid, 0u); return *this; } - LidSet scan(uint32_t count, uint32_t compactLidLimit, uint32_t maxDocsToScan = 10) { + LidSet scan(uint32_t count, uint32_t compactLidLimit) { + if (!_itr) { + _itr = std::make_unique<DocumentScanIterator>(_metaStore); + } LidSet retval; for (uint32_t i = 0; i < count; ++i) { - retval.insert(next(compactLidLimit, maxDocsToScan, false)); - EXPECT_TRUE(_itr.valid()); + uint32_t lid = next(compactLidLimit, false); + retval.insert(lid); + EXPECT_TRUE(_itr->valid() || lid <= compactLidLimit); } - EXPECT_EQUAL(0u, next(compactLidLimit, maxDocsToScan, false)); - EXPECT_FALSE(_itr.valid()); + EXPECT_EQUAL(0u, next(compactLidLimit, false)); + EXPECT_FALSE(_itr->valid()); return retval; } - uint32_t next(uint32_t compactLidLimit, uint32_t maxDocsToScan = 10, bool retry = false) { - return _itr.next(compactLidLimit, maxDocsToScan, retry).lid; + uint32_t next(uint32_t compactLidLimit, bool retry = false) { + if (!_itr) { + _itr = std::make_unique<DocumentScanIterator>(_metaStore); + } + return _itr->next(compactLidLimit, retry).lid; } }; @@ -74,24 +81,11 @@ TEST_F("require that only lids > lid limit are returned", Fixture) assertLidSet({5,6,7,8}, f.scan(4, 4)); } -TEST_F("require that max docs to scan (1) are taken into consideration", Fixture) -{ - f.add({1,2,3,4,5,6,7,8}); - assertLidSet({0,5,6,7,8}, f.scan(8, 4, 1)); -} - -TEST_F("require that max docs to scan (2) are taken into consideration", Fixture) -{ - f.add({1,2,3,4,5,6,7,8}); - // scan order is: 8, {2,4}, 7, {5,3}, {1,6} (5 scans total) - assertLidSet({0,7,8}, f.scan(5, 6, 2)); -} - TEST_F("require that we start scan at previous doc if retry is set", Fixture) { f.add({1,2,3,4,5,6,7,8}); - uint32_t lid1 = f.next(4, 10, false); - uint32_t lid2 = f.next(4, 10, true); + uint32_t lid1 = f.next(4, false); + uint32_t lid2 = f.next(4, true); EXPECT_EQUAL(lid1, lid2); } diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 4f36608aace..c31141a943c 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -31,7 +31,6 @@ constexpr uint32_t ALLOWED_LID_BLOAT = 1; constexpr double ALLOWED_LID_BLOAT_FACTOR = 0.3; constexpr double REMOVE_BATCH_BLOCK_RATE = 1.0 / 21.0; constexpr double REMOVE_BLOCK_RATE = 1.0 / 20.0; -constexpr uint32_t MAX_DOCS_TO_SCAN = 100; constexpr double RESOURCE_LIMIT_FACTOR = 1.0; constexpr uint32_t MAX_OUTSTANDING_MOVE_OPS = 10; const vespalib::string DOC_ID = "id:test:searchdocument::0"; @@ -52,11 +51,11 @@ struct MyScanIterator : public IDocumentScanIterator { bool valid() const override { return _validItr; } - search::DocumentMetaData next(uint32_t compactLidLimit, uint32_t maxDocsToScan, bool retry) override { + search::DocumentMetaData next(uint32_t compactLidLimit, bool retry) override { if (!retry && _itr != _lids.begin()) { ++_itr; } - for (uint32_t i = 0; i < maxDocsToScan && _itr != _lids.end() && (*_itr) <= compactLidLimit; ++i, ++_itr) {} + for (; _itr != _lids.end() && (*_itr) <= compactLidLimit; ++_itr) {} if (_itr != _lids.end()) { uint32_t lid = *_itr; if (lid > compactLidLimit) { @@ -75,7 +74,6 @@ struct MyHandler : public ILidSpaceCompactionHandler { mutable uint32_t _moveFromLid; mutable uint32_t _moveToLid; uint32_t _handleMoveCnt; - uint32_t _wantedSubDbId; uint32_t _wantedLidLimit; mutable uint32_t _iteratorCnt; bool _storeMoveDoneContexts; @@ -137,7 +135,6 @@ struct MyHandler : public ILidSpaceCompactionHandler { } } void handleCompactLidSpace(const CompactLidSpaceOperation &op, std::shared_ptr<IDestructorCallback>) override { - _wantedSubDbId = op.getSubDbId(); _wantedLidLimit = op.getLidLimit(); } }; @@ -147,7 +144,6 @@ MyHandler::MyHandler(bool storeMoveDoneContexts) _moveFromLid(0), _moveToLid(0), _handleMoveCnt(0), - _wantedSubDbId(0), _wantedLidLimit(0), _iteratorCnt(0), _storeMoveDoneContexts(storeMoveDoneContexts), @@ -281,7 +277,6 @@ struct JobTestBase : public ::testing::Test { } void init(uint32_t allowedLidBloat = ALLOWED_LID_BLOAT, double allowedLidBloatFactor = ALLOWED_LID_BLOAT_FACTOR, - uint32_t maxDocsToScan = MAX_DOCS_TO_SCAN, double resourceLimitFactor = RESOURCE_LIMIT_FACTOR, vespalib::duration interval = JOB_DELAY, bool nodeRetired = false, @@ -292,7 +287,7 @@ struct JobTestBase : public ::testing::Test { allowedLidBloatFactor, REMOVE_BATCH_BLOCK_RATE, REMOVE_BLOCK_RATE, - false, maxDocsToScan), + false), *_handler, _storer, _frozenHandler, _diskMemUsageNotifier, BlockableMaintenanceJobConfig(resourceLimitFactor, maxOutstandingMoveOps), _clusterStateHandler, nodeRetired); @@ -386,19 +381,18 @@ struct JobTest : public JobTestBase { {} void init(uint32_t allowedLidBloat = ALLOWED_LID_BLOAT, double allowedLidBloatFactor = ALLOWED_LID_BLOAT_FACTOR, - uint32_t maxDocsToScan = MAX_DOCS_TO_SCAN, double resourceLimitFactor = RESOURCE_LIMIT_FACTOR, vespalib::duration interval = JOB_DELAY, bool nodeRetired = false, uint32_t maxOutstandingMoveOps = MAX_OUTSTANDING_MOVE_OPS) { - JobTestBase::init(allowedLidBloat, allowedLidBloatFactor, maxDocsToScan, resourceLimitFactor, interval, nodeRetired, maxOutstandingMoveOps); + JobTestBase::init(allowedLidBloat, allowedLidBloatFactor, resourceLimitFactor, interval, nodeRetired, maxOutstandingMoveOps); _jobRunner = std::make_unique<MyDirectJobRunner>(*_job); } void init_with_interval(vespalib::duration interval) { - init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, MAX_DOCS_TO_SCAN, RESOURCE_LIMIT_FACTOR, interval); + init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, interval); } void init_with_node_retired(bool retired) { - init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, MAX_DOCS_TO_SCAN, RESOURCE_LIMIT_FACTOR, JOB_DELAY, retired); + init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, retired); } }; @@ -490,16 +484,6 @@ TEST_F(JobTest, job_is_blocked_if_trying_to_move_document_for_frozen_bucket) EXPECT_FALSE(_job->isBlocked()); } -TEST_F(JobTest, job_handles_invalid_document_meta_data_when_max_docs_are_scanned) -{ - init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 3); - setupOneDocumentToCompact(); - EXPECT_FALSE(run()); // does not find 9 in first scan - assertNoWorkDone(); - EXPECT_FALSE(run()); // move 9 -> 2 - assertOneDocumentCompacted(); -} - TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) { init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 3); @@ -513,11 +497,8 @@ TEST_F(JobTest, job_can_restart_documents_scan_if_lid_bloat_is_still_to_large) // We simulate that the set of used docs have changed between these 2 runs EXPECT_FALSE(run()); // move 9 -> 2 endScan(); - assertJobContext(2, 9, 1, 0, 0); - EXPECT_EQ(2u, _handler->_iteratorCnt); - EXPECT_FALSE(run()); // does not find 8 in first scan - EXPECT_FALSE(run()); // move 8 -> 3 assertJobContext(3, 8, 2, 0, 0); + EXPECT_EQ(2u, _handler->_iteratorCnt); endScan().compact(); assertJobContext(3, 8, 2, 7, 1); } @@ -602,7 +583,7 @@ TEST_F(JobTest, ending_resource_starvation_resumes_lid_space_compaction) TEST_F(JobTest, resource_limit_factor_adjusts_limit) { - init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, MAX_DOCS_TO_SCAN, 1.05); + init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, 1.05); setupOneDocumentToCompact(); _diskMemUsageNotifier.notify({{100, 0}, {100, 101}}); EXPECT_FALSE(run()); // scan @@ -729,7 +710,7 @@ struct MaxOutstandingJobTest : public JobTest { runner() {} void init(uint32_t maxOutstandingMoveOps) { - JobTest::init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, MAX_DOCS_TO_SCAN, + JobTest::init(ALLOWED_LID_BLOAT, ALLOWED_LID_BLOAT_FACTOR, RESOURCE_LIMIT_FACTOR, JOB_DELAY, false, maxOutstandingMoveOps); runner = std::make_unique<MyCountJobRunner>(*_job); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 0d5d8ad3144..8bec6f9dd68 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -3,6 +3,7 @@ #include "persistenceengine.h" #include "ipersistenceengineowner.h" #include "transport_latch.h" +#include <vespa/persistence/spi/bucketexecutor.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> @@ -18,6 +19,7 @@ LOG_SETUP(".proton.persistenceengine.persistenceengine"); using document::Document; using document::DocumentId; using storage::spi::BucketChecksum; +using storage::spi::BucketExecutor; using storage::spi::BucketIdListResult; using storage::spi::BucketInfo; using storage::spi::BucketInfoResult; @@ -737,4 +739,28 @@ PersistenceEngine::getWLock() const return WriteGuard(_rwMutex); } +namespace { + +class SyncExecutorOnDestruction : public vespalib::IDestructorCallback { +public: + explicit SyncExecutorOnDestruction(std::shared_ptr<BucketExecutor> executor) : _executor(std::move(executor)) { } + ~SyncExecutorOnDestruction() override { + if (_executor) { + _executor->sync(); + } + } +private: + std::shared_ptr<BucketExecutor> _executor; +}; + +} + +std::unique_ptr<vespalib::IDestructorCallback> +PersistenceEngine::register_executor(std::shared_ptr<BucketExecutor> executor) +{ + assert(_bucket_executor.expired()); + _bucket_executor = executor; + return std::make_unique<SyncExecutorOnDestruction>(executor); +} + } // storage diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index 659156fdea0..b5a99525575 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -39,6 +39,7 @@ private: using TimestampList = storage::spi::TimestampList; using UpdateResult = storage::spi::UpdateResult; using OperationComplete = storage::spi::OperationComplete; + using BucketExecutor = storage::spi::BucketExecutor; struct IteratorEntry { PersistenceHandlerSequence handler_sequence; @@ -73,6 +74,7 @@ private: mutable ExtraModifiedBuckets _extraModifiedBuckets; mutable std::shared_mutex _rwMutex; std::shared_ptr<ResourceUsageTracker> _resource_usage_tracker; + std::weak_ptr<BucketExecutor> _bucket_executor; using ReadGuard = std::shared_lock<std::shared_mutex>; using WriteGuard = std::unique_lock<std::shared_mutex>; @@ -116,12 +118,14 @@ public: Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; std::unique_ptr<vespalib::IDestructorCallback> register_resource_usage_listener(IResourceUsageListener& listener) override; + std::unique_ptr<vespalib::IDestructorCallback> register_executor(std::shared_ptr<BucketExecutor>) override; void destroyIterators(); void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler); void populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler); WriteGuard getWLock() const; ResourceUsageTracker &get_resource_usage_tracker() noexcept { return *_resource_usage_tracker; } + std::shared_ptr<BucketExecutor> get_bucket_executor() noexcept { return _bucket_executor.lock(); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp index 24965db5919..f365b6ae02d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp @@ -55,8 +55,7 @@ DocumentDBLidSpaceCompactionConfig::DocumentDBLidSpaceCompactionConfig() _allowedLidBloatFactor(1.0), _remove_batch_block_rate(0.5), _remove_block_rate(100), - _disabled(false), - _maxDocsToScan(10000) + _disabled(false) { } @@ -65,16 +64,14 @@ DocumentDBLidSpaceCompactionConfig::DocumentDBLidSpaceCompactionConfig(vespalib: double allowedLidBloatFactor, double remove_batch_block_rate, double remove_block_rate, - bool disabled, - uint32_t maxDocsToScan) + bool disabled) : _delay(std::min(MAX_DELAY_SEC, interval)), _interval(interval), _allowedLidBloat(allowedLidBloat), _allowedLidBloatFactor(allowedLidBloatFactor), _remove_batch_block_rate(remove_batch_block_rate), _remove_block_rate(remove_block_rate), - _disabled(disabled), - _maxDocsToScan(maxDocsToScan) + _disabled(disabled) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 80064b7b5d7..731add1f62c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -50,7 +50,6 @@ private: double _remove_batch_block_rate; double _remove_block_rate; bool _disabled; - uint32_t _maxDocsToScan; public: DocumentDBLidSpaceCompactionConfig(); @@ -59,8 +58,7 @@ public: double allowwedLidBloatFactor, double remove_batch_block_rate, double remove_block_rate, - bool disabled, - uint32_t maxDocsToScan = 10000); + bool disabled); static DocumentDBLidSpaceCompactionConfig createDisabled(); bool operator==(const DocumentDBLidSpaceCompactionConfig &rhs) const; @@ -71,7 +69,6 @@ public: double get_remove_batch_block_rate() const { return _remove_batch_block_rate; } double get_remove_block_rate() const { return _remove_block_rate; } bool isDisabled() const { return _disabled; } - uint32_t getMaxDocsToScan() const { return _maxDocsToScan; } }; class BlockableMaintenanceJobConfig { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp index cbab4b72971..290fbc79951 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp @@ -11,8 +11,7 @@ typedef IDocumentMetaStore::Iterator Iterator; DocumentScanIterator::DocumentScanIterator(const IDocumentMetaStore &metaStore) : _metaStore(metaStore), - _lastGid(), - _lastGidValid(false), + _lastLid(_metaStore.getCommittedDocIdLimit()), _itrValid(true) { } @@ -24,27 +23,19 @@ DocumentScanIterator::valid() const } DocumentMetaData -DocumentScanIterator::next(uint32_t compactLidLimit, - uint32_t maxDocsToScan, - bool retry) +DocumentScanIterator::next(uint32_t compactLidLimit, bool retry) { - Iterator itr = (_lastGidValid ? - (retry ? _metaStore.lowerBound(_lastGid) : _metaStore.upperBound(_lastGid)) - : _metaStore.begin()); - uint32_t i = 1; // We have already 'scanned' a document when creating the iterator - for (; i < maxDocsToScan && itr.valid() && itr.getKey().get_lid() <= compactLidLimit; ++i, ++itr) {} - if (itr.valid()) { - uint32_t lid = itr.getKey().get_lid(); - const RawDocumentMetaData &metaData = _metaStore.getRawMetaData(lid); - _lastGid = metaData.getGid(); - _lastGidValid = true; - if (lid > compactLidLimit) { - return DocumentMetaData(lid, metaData.getTimestamp(), - metaData.getBucketId(), metaData.getGid()); + if (!retry) { + --_lastLid; + } + for (uint32_t i(0); _lastLid > compactLidLimit; ++i, --_lastLid) { + if (_metaStore.validLid(_lastLid)) { + const RawDocumentMetaData &metaData = _metaStore.getRawMetaData(_lastLid); + return DocumentMetaData(_lastLid, metaData.getTimestamp(), + metaData.getBucketId(), metaData.getGid()); } - } else { - _itrValid = false; } + _itrValid = (_lastLid > compactLidLimit) ; return DocumentMetaData(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h index fec547236d2..62dff0a7d85 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h @@ -15,19 +15,13 @@ class DocumentScanIterator : public IDocumentScanIterator { private: const IDocumentMetaStore &_metaStore; - document::GlobalId _lastGid; - bool _lastGidValid; + uint32_t _lastLid; bool _itrValid; public: DocumentScanIterator(const IDocumentMetaStore &_metaStore); - - // Implements IDocumentScanIterator - virtual bool valid() const override; - - virtual search::DocumentMetaData next(uint32_t compactLidLimit, - uint32_t maxDocsToScan, - bool retry) override; + bool valid() const override; + search::DocumentMetaData next(uint32_t compactLidLimit, bool retry) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h b/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h index 263d1e2945a..bcc52ffb475 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_document_scan_iterator.h @@ -14,7 +14,7 @@ struct IDocumentScanIterator { typedef std::unique_ptr<IDocumentScanIterator> UP; - virtual ~IDocumentScanIterator() {} + virtual ~IDocumentScanIterator() = default; /** * Returns false if we are certain there are no more documents to scan, true otherwise. @@ -24,16 +24,12 @@ struct IDocumentScanIterator /** * Returns the next document that has lid > compactLidLimit to be moved. - * Returns an invalid document if no documents satisfy the limit or - * if max documents are scanned. + * Returns an invalid document if no documents satisfy the limit. * * @param compactLidLimit The returned document must have lid larger than this limit. - * @param maxDocsToScan The maximum documents to scan before returning. * @param retry Whether we should start the scan with the previous returned document. */ - virtual search::DocumentMetaData next(uint32_t compactLidLimit, - uint32_t maxDocsToScan, - bool retry) = 0; + virtual search::DocumentMetaData next(uint32_t compactLidLimit, bool retry) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 43ba91e2ef1..1165c8e345e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -35,9 +35,7 @@ LidSpaceCompactionJob::shouldRestartScanDocuments(const LidUsageStats &stats) co DocumentMetaData LidSpaceCompactionJob::getNextDocument(const LidUsageStats &stats) { - DocumentMetaData document = - _scanItr->next(std::max(stats.getLowestFreeLid(), stats.getUsedLids()), - _cfg.getMaxDocsToScan(), _retryFrozenDocument); + DocumentMetaData document = _scanItr->next(std::max(stats.getLowestFreeLid(), stats.getUsedLids()), _retryFrozenDocument); _retryFrozenDocument = false; return document; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 0e6d20115a0..4396224fb01 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -31,7 +31,7 @@ namespace vespalib { class StateServer; } namespace search::transactionlog { class TransLogServerApp; } namespace metrics { class MetricLockGuard; } -namespace storage::spi { class PersistenceProvider; } +namespace storage::spi { struct PersistenceProvider; } namespace proton { diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp index 046e2dcdc2b..bc510077a4d 100644 --- a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.cpp @@ -162,7 +162,7 @@ StreamedValueStore::TensorBufferType::TensorBufferType() } void -StreamedValueStore::TensorBufferType::cleanHold(void* buffer, size_t offset, size_t num_elems, CleanContext clean_ctx) +StreamedValueStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) { TensorEntry::SP* elem = static_cast<TensorEntry::SP*>(buffer) + offset; for (size_t i = 0; i < num_elems; ++i) { diff --git a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h index 1df860f4007..a9353c37a06 100644 --- a/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h +++ b/searchlib/src/vespa/searchlib/tensor/streamed_value_store.h @@ -55,7 +55,7 @@ private: using CleanContext = typename ParentType::CleanContext; public: TensorBufferType(); - virtual void cleanHold(void* buffer, size_t offset, size_t num_elems, CleanContext clean_ctx) override; + virtual void cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) override; }; TensorStoreType _concrete_store; const vespalib::eval::ValueType _tensor_type; diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 0a756d5913f..11d0406fcc5 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -66,6 +66,12 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> </dependencies> <build> diff --git a/storage/src/tests/common/testnodestateupdater.cpp b/storage/src/tests/common/testnodestateupdater.cpp index c4afda1a5ad..27f21a31768 100644 --- a/storage/src/tests/common/testnodestateupdater.cpp +++ b/storage/src/tests/common/testnodestateupdater.cpp @@ -10,7 +10,8 @@ TestNodeStateUpdater::TestNodeStateUpdater(const lib::NodeType& type) _current(new lib::NodeState(type, lib::State::UP)), _clusterStateBundle(std::make_shared<const lib::ClusterStateBundle>(lib::ClusterState())), _listeners(), - _explicit_node_state_reply_send_invocations(0) + _explicit_node_state_reply_send_invocations(0), + _requested_almost_immediate_node_state_replies(0) { } TestNodeStateUpdater::~TestNodeStateUpdater() = default; diff --git a/storage/src/tests/common/testnodestateupdater.h b/storage/src/tests/common/testnodestateupdater.h index e0c636d2715..eb15b97a37f 100644 --- a/storage/src/tests/common/testnodestateupdater.h +++ b/storage/src/tests/common/testnodestateupdater.h @@ -19,6 +19,7 @@ struct TestNodeStateUpdater : public NodeStateUpdater std::shared_ptr<const lib::ClusterStateBundle> _clusterStateBundle; std::vector<StateListener*> _listeners; size_t _explicit_node_state_reply_send_invocations; + size_t _requested_almost_immediate_node_state_replies; public: explicit TestNodeStateUpdater(const lib::NodeType& type); @@ -37,6 +38,10 @@ public: ++_explicit_node_state_reply_send_invocations; } + void request_almost_immediate_node_state_replies() override { + ++_requested_almost_immediate_node_state_replies; + } + void setCurrentNodeState(const lib::NodeState& state) { _current = std::make_shared<lib::NodeState>(state); } @@ -47,6 +52,10 @@ public: size_t explicit_node_state_reply_send_invocations() const noexcept { return _explicit_node_state_reply_send_invocations; } + + size_t requested_almost_immediate_node_state_replies() const noexcept { + return _requested_almost_immediate_node_state_replies; + } }; } // storage diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 433f535546f..3742879bd30 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -35,7 +35,7 @@ namespace storage { -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } class StorageBucketDBInitializer; DEFINE_PRIMITIVE_WRAPPER(uint16_t, NodeIndex); diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 1518c8594aa..924678a6cd0 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -151,6 +151,12 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil { return cb; } + void set_up_distributor_with_feed_blocked_state() { + setup_distributor(2, 2, + lib::ClusterStateBundle(lib::ClusterState("distributor:1 storage:2"), + {}, {true, "full disk"}, false)); + } + }; TwoPhaseUpdateOperationTest::TwoPhaseUpdateOperationTest() = default; @@ -1091,6 +1097,17 @@ TEST_F(TwoPhaseUpdateOperationTest, update_gets_are_sent_with_strong_consistency EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Strong); } +TEST_F(TwoPhaseUpdateOperationTest, operation_is_rejected_in_safe_path_if_feed_is_blocked) { + set_up_distributor_with_feed_blocked_state(); + auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas to trigger safe path + cb->start(_sender, framework::MilliSecTime(0)); + + EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(NO_SPACE, External feed is blocked due to resource exhaustion: full disk)", + _sender.getLastReply(true)); +} + struct ThreePhaseUpdateTest : TwoPhaseUpdateOperationTest {}; TEST_F(ThreePhaseUpdateTest, metadata_only_gets_are_sent_if_3phase_update_enabled) { diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index bbde377fdec..21a94a3e957 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -209,6 +209,12 @@ PersistenceProviderWrapper::register_resource_usage_listener(spi::IResourceUsage return _spi.register_resource_usage_listener(listener); } +std::unique_ptr<vespalib::IDestructorCallback> +PersistenceProviderWrapper::register_executor(std::shared_ptr<spi::BucketExecutor> executor) +{ + return _spi.register_executor(std::move(executor)); +} + spi::Result PersistenceProviderWrapper::removeEntry(const spi::Bucket& bucket, spi::Timestamp timestamp, diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index b07f9c5e0f5..085a60c0e86 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -110,6 +110,7 @@ public: const spi::Bucket& target, spi::Context&) override; spi::Result removeEntry(const spi::Bucket&, spi::Timestamp, spi::Context&) override; std::unique_ptr<vespalib::IDestructorCallback> register_resource_usage_listener(spi::IResourceUsageListener& listener) override; + std::unique_ptr<vespalib::IDestructorCallback> register_executor(std::shared_ptr<spi::BucketExecutor>) override; }; } // storage diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index 7bd74b83786..33d20e97cb1 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -10,11 +10,13 @@ vespa_add_executable(storage_filestorage_gtest_runner_app TEST modifiedbucketcheckertest.cpp operationabortingtest.cpp sanitycheckeddeletetest.cpp + service_layer_host_info_reporter_test.cpp singlebucketjointest.cpp gtest_runner.cpp DEPENDS storage storageapi + storage_testhostreporter storage_testpersistence_common GTest::GTest ) diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 74ad5d7f2ce..acccbb8b9b9 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -14,6 +14,7 @@ #include <vespa/fastos/file.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/persistence/spi/test.h> +#include <vespa/persistence/spi/bucket_tasks.h> #include <vespa/storage/bucketdb/bucketmanager.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> #include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> @@ -24,6 +25,7 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/gate.h> #include <atomic> #include <thread> @@ -35,6 +37,7 @@ using document::Document; using namespace storage::api; using storage::spi::test::makeSpiBucket; using document::test::makeDocumentBucket; +using vespalib::IDestructorCallback; using namespace ::testing; #define ASSERT_SINGLE_REPLY(replytype, reply, link, time) \ @@ -409,6 +412,50 @@ TEST_F(FileStorManagerTest, put) { } } +TEST_F(FileStorManagerTest, running_task_against_unknown_bucket_fails) { + TestFileStorComponents c(*this); + + setClusterState("storage:3 distributor:3"); + EXPECT_TRUE(getDummyPersistence().getClusterState().nodeUp()); + + auto executor = getDummyPersistence().get_bucket_executor(); + ASSERT_TRUE(executor); + + spi::Bucket b1 = makeSpiBucket(document::BucketId(1)); + std::atomic<size_t> numInvocations(0); + auto response = executor->execute(b1, spi::makeBucketTask([&numInvocations](const spi::Bucket &, std::shared_ptr<IDestructorCallback>) { + numInvocations++; + })); + ASSERT_TRUE(response); + EXPECT_EQ(0, numInvocations); + response->run(spi::Bucket(), {}); + EXPECT_EQ(1, numInvocations); +} + +TEST_F(FileStorManagerTest, running_task_against_existing_bucket_works) { + TestFileStorComponents c(*this); + + setClusterState("storage:3 distributor:3"); + EXPECT_TRUE(getDummyPersistence().getClusterState().nodeUp()); + + auto executor = getDummyPersistence().get_bucket_executor(); + ASSERT_TRUE(executor); + + spi::Bucket b1 = makeSpiBucket(document::BucketId(1)); + + createBucket(b1.getBucketId()); + + std::atomic<size_t> numInvocations(0); + vespalib::Gate gate; + auto response = executor->execute(b1, spi::makeBucketTask([&numInvocations, &gate](const spi::Bucket &, std::shared_ptr<IDestructorCallback>) { + numInvocations++; + gate.countDown(); + })); + EXPECT_FALSE(response); + gate.await(); + EXPECT_EQ(1, numInvocations); +} + TEST_F(FileStorManagerTest, state_change) { TestFileStorComponents c(*this); diff --git a/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp b/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp new file mode 100644 index 00000000000..cf9d380fb13 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp @@ -0,0 +1,91 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h> +#include <tests/common/hostreporter/util.h> +#include <tests/common/testnodestateupdater.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <iostream> + +namespace storage { + +using spi::ResourceUsage; + +namespace { + +double +get_usage_element(const vespalib::Slime& root, const vespalib::string& label) +{ + return root.get()["content-node"]["resource-usage"][label]["usage"].asDouble(); +} + +} + +struct ServiceLayerHostInfoReporterTest : ::testing::Test { + + TestNodeStateUpdater _state_manager; + ServiceLayerHostInfoReporter _reporter; + + ServiceLayerHostInfoReporterTest(); + ~ServiceLayerHostInfoReporterTest(); + + void notify(double disk_usage, double memory_usage) { + auto& listener = static_cast<spi::IResourceUsageListener&>(_reporter); + listener.update_resource_usage(ResourceUsage(disk_usage, memory_usage)); + } + + size_t requested_almost_immediate_replies() { return _state_manager.requested_almost_immediate_node_state_replies(); } + ResourceUsage get_old_usage() { return _reporter.get_old_resource_usage(); } + ResourceUsage get_usage() { return _reporter.get_usage(); } + ResourceUsage get_slime_usage() { + vespalib::Slime root; + util::reporterToSlime(_reporter, root); + return ResourceUsage(get_usage_element(root, "disk"), get_usage_element(root, "memory")); + } +}; + +ServiceLayerHostInfoReporterTest::ServiceLayerHostInfoReporterTest() + : _state_manager(lib::NodeType::STORAGE), + _reporter(_state_manager) +{ +} + +ServiceLayerHostInfoReporterTest::~ServiceLayerHostInfoReporterTest() = default; + +TEST_F(ServiceLayerHostInfoReporterTest, request_almost_immediate_node_state_as_needed) +{ + EXPECT_EQ(0, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.0, 0.0), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.0, 0.0), get_usage()); + notify(0.5, 0.4); + EXPECT_EQ(1, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.5, 0.4), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.5, 0.4), get_usage()); + notify(0.501, 0.401); + EXPECT_EQ(1, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.5, 0.4), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.501, 0.401), get_usage()); + notify(0.8, 0.4); + EXPECT_EQ(2, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.8, 0.4), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.8, 0.4), get_usage()); + notify(0.8, 0.7); + EXPECT_EQ(3, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.8, 0.7), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.8, 0.7), get_usage()); + notify(0.799, 0.699); + EXPECT_EQ(3, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.8, 0.7), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.799, 0.699), get_usage()); +} + +TEST_F(ServiceLayerHostInfoReporterTest, json_report_generated) +{ + EXPECT_EQ(ResourceUsage(0.0, 0.0), get_slime_usage()); + notify(0.5, 0.4); + EXPECT_EQ(ResourceUsage(0.5, 0.4), get_slime_usage()); +} + +} diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index b55e62d5fd3..1a9882bd0fa 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -316,6 +316,21 @@ TEST_F(StateManagerTest, immediate_node_state_replying_is_tracked_per_controller ASSERT_EQ(0, _upper->getNumReplies()); } +TEST_F(StateManagerTest, request_almost_immediate_replies_triggers_fast_reply) +{ + mark_reported_node_state_up(); + mark_reply_observed_from_n_controllers(1); + auto before = std::chrono::steady_clock::now(); + for (size_t pass = 0; pass < 100; ++pass) { + send_down_get_node_state_request(0); + _manager->request_almost_immediate_node_state_replies(); + _upper->waitForMessage(api::MessageType::GETNODESTATE_REPLY, 2); + clear_sent_replies(); + } + auto after = std::chrono::steady_clock::now(); + ASSERT_GT(10s, after - before); +} + TEST_F(StateManagerTest, activation_command_is_bounced_with_current_cluster_state_version) { force_current_cluster_state_version(12345); diff --git a/storage/src/vespa/storage/common/messagebucket.cpp b/storage/src/vespa/storage/common/messagebucket.cpp index 61283fd3d04..3aa90138f36 100644 --- a/storage/src/vespa/storage/common/messagebucket.cpp +++ b/storage/src/vespa/storage/common/messagebucket.cpp @@ -66,6 +66,8 @@ getStorageMessageBucket(const api::StorageMessage& msg) return static_cast<const ReadBucketInfo&>(msg).getBucket(); case RecheckBucketInfoCommand::ID: return static_cast<const RecheckBucketInfoCommand&>(msg).getBucket(); + case RunTaskCommand::ID: + return static_cast<const RunTaskCommand&>(msg).getBucket(); default: break; } diff --git a/storage/src/vespa/storage/common/nodestateupdater.h b/storage/src/vespa/storage/common/nodestateupdater.h index 60f4213fe54..da7cb72e321 100644 --- a/storage/src/vespa/storage/common/nodestateupdater.h +++ b/storage/src/vespa/storage/common/nodestateupdater.h @@ -71,6 +71,11 @@ struct NodeStateUpdater { * regardless of whether the reported state has changed. */ virtual void immediately_send_get_node_state_replies() = 0; + + /** + * Request almost immediate node state replies. + */ + virtual void request_almost_immediate_node_state_replies() = 0; }; } // storage diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index a00f7b4feb5..ceb25e099b2 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -92,8 +92,4 @@ use_content_node_btree_bucket_db bool default=true restart ## If non-zero, the bucket DB will be striped into 2^bits sub-databases, each handling ## a disjoint subset of the node's buckets, in order to reduce locking contention. ## Max value is unspecified, but will be clamped internally. -## WARNING: -## Setting this to a non-zero value requires the minimum split bit level in the cluster -## to be enforced, so only set this value if you know exactly what you're doing! -content_node_bucket_db_stripe_bits int default=0 restart - +content_node_bucket_db_stripe_bits int default=4 restart diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 80e7942c68e..362acdf18ec 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "twophaseupdateoperation.h" #include "getoperation.h" #include "putoperation.h" +#include "twophaseupdateoperation.h" #include "updateoperation.h" -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/distributor_bucket_space_repo.h> -#include <vespa/storageapi/message/persistence.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/select/parser.h> -#include <vespa/document/fieldset/fieldsets.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> @@ -197,6 +198,10 @@ TwoPhaseUpdateOperation::startFastPathUpdate(DistributorMessageSender& sender, s void TwoPhaseUpdateOperation::startSafePathUpdate(DistributorMessageSender& sender) { + if (_op_ctx.cluster_state_bundle().block_feed_in_cluster()) { + send_feed_blocked_error_reply(sender); + return; + } _mode = Mode::SLOW_PATH; auto get_operation = create_initial_safe_path_get_operation(); GetOperation& op = *get_operation; @@ -279,6 +284,15 @@ TwoPhaseUpdateOperation::sendLostOwnershipTransientErrorReply(DistributorMessage } void +TwoPhaseUpdateOperation::send_feed_blocked_error_reply(DistributorMessageSender& sender) +{ + sendReplyWithResult(sender, + api::ReturnCode(api::ReturnCode::NO_SPACE, + "External feed is blocked due to resource exhaustion: " + + _op_ctx.cluster_state_bundle().feed_block()->description())); +} + +void TwoPhaseUpdateOperation::schedulePutsWithUpdatedDocument(std::shared_ptr<document::Document> doc, api::Timestamp putTimestamp, DistributorMessageSender& sender) { diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h index af45932b530..d353498c8e3 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h @@ -104,6 +104,7 @@ private: void startSafePathUpdate(DistributorMessageSender&); bool lostBucketOwnershipBetweenPhases() const; void sendLostOwnershipTransientErrorReply(DistributorMessageSender&); + void send_feed_blocked_error_reply(DistributorMessageSender& sender); void schedulePutsWithUpdatedDocument( std::shared_ptr<document::Document>, api::Timestamp, diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index 4b105f6688f..b8ed6b8ec91 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -6,6 +6,7 @@ #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> +#include <vespa/vespalib/util/destructor_callbacks.h> namespace storage { @@ -96,6 +97,18 @@ AsyncHandler::AsyncHandler(const PersistenceUtil & env, spi::PersistenceProvider {} MessageTracker::UP +AsyncHandler::handleRunTask(RunTaskCommand& cmd, MessageTracker::UP tracker) const { + auto task = makeResultTask([tracker = std::move(tracker)](spi::Result::UP response) { + tracker->checkForError(*response); + tracker->sendReply(); + }); + spi::Bucket bucket(cmd.getBucket()); + auto onDone = std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task)); + cmd.task().run(bucket, std::make_shared<vespalib::KeepAlive<decltype(onDone)>>(std::move(onDone))); + return tracker; +} + +MessageTracker::UP AsyncHandler::handlePut(api::PutCommand& cmd, MessageTracker::UP trackerUP) const { MessageTracker & tracker = *trackerUP; diff --git a/storage/src/vespa/storage/persistence/asynchandler.h b/storage/src/vespa/storage/persistence/asynchandler.h index 92bf72e7c51..2d6b37a1cdd 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.h +++ b/storage/src/vespa/storage/persistence/asynchandler.h @@ -2,6 +2,7 @@ #pragma once #include "types.h" +#include "messages.h" #include <vespa/storageapi/message/persistence.h> namespace document { class BucketIdFactory; } @@ -25,6 +26,7 @@ public: MessageTrackerUP handlePut(api::PutCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleRemove(api::RemoveCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleUpdate(api::UpdateCommand& cmd, MessageTrackerUP tracker) const; + MessageTrackerUP handleRunTask(RunTaskCommand & cmd, MessageTrackerUP tracker) const; static bool is_async_message(api::MessageType::Id type_id) noexcept; private: static bool tasConditionExists(const api::TestAndSetCommand & cmd); diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.h b/storage/src/vespa/storage/persistence/bucketprocessor.h index 5b96696475e..c4ebcce6165 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.h +++ b/storage/src/vespa/storage/persistence/bucketprocessor.h @@ -11,7 +11,7 @@ #include <vespa/persistence/spi/context.h> namespace document { class FieldSet; } -namespace storage::spi { class PersistenceProvider; } +namespace storage::spi { struct PersistenceProvider; } namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt index 2fd54930f77..537470a5be0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt @@ -8,5 +8,6 @@ vespa_add_library(storage_filestorpersistence OBJECT merge_handler_metrics.cpp mergestatus.cpp modifiedbucketchecker.cpp + service_layer_host_info_reporter.cpp DEPENDS ) diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 0f9c9894615..67fa22ada03 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -7,6 +7,7 @@ #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/vdslib/state/cluster_state_bundle.h> +#include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/common/messagebucket.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> @@ -76,15 +77,16 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _use_async_message_handling_on_schedule(false), _metrics(std::make_unique<FileStorMetrics>()), _closed(false), - _lock() + _lock(), + _host_info_reporter(_component.getStateUpdater()) { _configFetcher.subscribe(configUri.getConfigId(), this); _configFetcher.start(); _component.registerMetric(*_metrics); _component.registerStatusPage(*this); _component.getStateUpdater().addStateListener(*this); + hostInfoReporterRegistrar.registerReporter(&_host_info_reporter); propagateClusterStates(); - (void) hostInfoReporterRegistrar; } FileStorManager::~FileStorManager() @@ -972,7 +974,11 @@ void FileStorManager::initialize_bucket_databases_from_provider() { std::unique_ptr<spi::BucketTask> FileStorManager::execute(const spi::Bucket &bucket, std::unique_ptr<spi::BucketTask> task) { - (void) bucket; + StorBucketDatabase::WrappedEntry entry(_component.getBucketDatabase(bucket.getBucketSpace()).get( + bucket.getBucketId(), "FileStorManager::execute")); + if (entry.exist()) { + _filestorHandler->schedule(std::make_shared<RunTaskCommand>(bucket, std::move(task))); + } return task; } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 489dd1f97e9..ae298d70a29 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -9,6 +9,7 @@ #pragma once #include "filestorhandler.h" +#include "service_layer_host_info_reporter.h" #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/document/bucket/bucketid.h> @@ -36,7 +37,7 @@ namespace api { class StorageReply; class BucketCommand; } -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } struct FileStorManagerTest; class ReadBucketList; @@ -75,6 +76,7 @@ class FileStorManager : public StorageLinkQueued, bool _closed; std::mutex _lock; std::unique_ptr<vespalib::IDestructorCallback> _bucketExecutorRegistration; + ServiceLayerHostInfoReporter _host_info_reporter; public: FileStorManager(const config::ConfigUri &, spi::PersistenceProvider&, diff --git a/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp new file mode 100644 index 00000000000..784aa16bf02 --- /dev/null +++ b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp @@ -0,0 +1,77 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "service_layer_host_info_reporter.h" +#include <vespa/storage/common/nodestateupdater.h> +#include <cmath> + +namespace storage { + +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; + +namespace { + +constexpr double diff_slack = 0.01; + +void write_usage(vespalib::JsonStream& output, const vespalib::string &label, double value) +{ + output << label << Object(); + output << "usage" << value; + output << End(); +} + +bool want_immediate_report(const spi::ResourceUsage& old_resource_usage, const spi::ResourceUsage& resource_usage) +{ + auto disk_usage_diff = fabs(resource_usage.get_disk_usage() - old_resource_usage.get_disk_usage()); + auto memory_usage_diff = fabs(resource_usage.get_memory_usage() - old_resource_usage.get_memory_usage()); + return (disk_usage_diff > diff_slack || memory_usage_diff > diff_slack); +} + +} + +ServiceLayerHostInfoReporter::ServiceLayerHostInfoReporter(NodeStateUpdater& node_state_updater) + : HostReporter(), + spi::ResourceUsageListener(), + _node_state_updater(node_state_updater), + _lock(), + _old_resource_usage() +{ +} + +ServiceLayerHostInfoReporter::~ServiceLayerHostInfoReporter() +{ + spi::ResourceUsageListener::reset(); // detach +} + +void +ServiceLayerHostInfoReporter::update_resource_usage(const spi::ResourceUsage& resource_usage) +{ + bool immediate_report = want_immediate_report(_old_resource_usage, resource_usage); + if (immediate_report) { + _old_resource_usage = resource_usage; + } + { + std::lock_guard guard(_lock); + spi::ResourceUsageListener::update_resource_usage(resource_usage); + } + if (immediate_report) { + _node_state_updater.request_almost_immediate_node_state_replies(); + } +} + +void +ServiceLayerHostInfoReporter::report(vespalib::JsonStream& output) +{ + output << "content-node" << Object(); + output << "resource-usage" << Object(); + { + std::lock_guard guard(_lock); + auto& usage = get_usage(); + write_usage(output, "memory", usage.get_memory_usage()); + write_usage(output, "disk", usage.get_disk_usage()); + } + output << End(); + output << End(); +} + +} diff --git a/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h new file mode 100644 index 00000000000..be0abc94987 --- /dev/null +++ b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/persistence/spi/resource_usage_listener.h> +#include <vespa/storage/common/hostreporter/hostreporter.h> +#include <mutex> + +namespace storage { + +struct NodeStateUpdater; + +/* + * Host info reporter for service layer that provides resource usage. + */ +class ServiceLayerHostInfoReporter : public HostReporter, + public spi::ResourceUsageListener +{ + NodeStateUpdater& _node_state_updater; + std::mutex _lock; + spi::ResourceUsage _old_resource_usage; + + void update_resource_usage(const spi::ResourceUsage& resource_usage) override; +public: + ServiceLayerHostInfoReporter(NodeStateUpdater& node_state_updater); + + ServiceLayerHostInfoReporter(const ServiceLayerHostInfoReporter&) = delete; + ServiceLayerHostInfoReporter& operator=(const ServiceLayerHostInfoReporter&) = delete; + ~ServiceLayerHostInfoReporter(); + + void report(vespalib::JsonStream& output) override; + const spi::ResourceUsage &get_old_resource_usage() noexcept { return _old_resource_usage; } +}; + +} diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index 90597a49ad7..7ccb3ee895d 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -2,6 +2,7 @@ #include "messages.h" #include <ostream> +#include <cassert> using document::BucketSpace; @@ -177,4 +178,43 @@ AbortBucketOperationsCommand::makeReply() { return std::make_unique<AbortBucketOperationsReply>(*this); } +std::unique_ptr<api::StorageReply> +RunTaskCommand::makeReply() { + return std::make_unique<RunTaskReply>(*this); +} + +RunTaskCommand::RunTaskCommand(const spi::Bucket &bucket, std::unique_ptr<spi::BucketTask> task) + : api::InternalCommand(ID), + _task(std::move(task)), + _bucket(bucket) +{ + assert(_task); +} + +RunTaskCommand::~RunTaskCommand() = default; + +void +RunTaskCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { + out << "RunTaskCommand(" << _bucket <<")"; + + if (verbose) { + out << " : "; + InternalCommand::print(out, true, indent); + } +} + +RunTaskReply::RunTaskReply(const RunTaskCommand& cmd) + : api::InternalReply(ID, cmd) +{} + +void +RunTaskReply::print(std::ostream& out, bool verbose, const std::string& indent) const { + out << "RunTaskReply()"; + + if (verbose) { + out << " : "; + InternalReply::print(out, true, indent); + } +} + } diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h index a465437ae21..043747d10d2 100644 --- a/storage/src/vespa/storage/persistence/messages.h +++ b/storage/src/vespa/storage/persistence/messages.h @@ -6,6 +6,7 @@ #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/selection.h> #include <vespa/persistence/spi/read_consistency.h> +#include <vespa/persistence/spi/bucketexecutor.h> namespace storage { @@ -16,7 +17,7 @@ private: uint32_t _maxByteSize; public: - static const uint32_t ID = 1001; + static constexpr uint32_t ID = 1001; typedef std::unique_ptr<GetIterCommand> UP; typedef std::shared_ptr<GetIterCommand> SP; @@ -50,7 +51,7 @@ private: public: typedef std::unique_ptr<GetIterReply> UP; typedef std::shared_ptr<GetIterReply> SP; - static const uint32_t ID = 1002; + static constexpr uint32_t ID = 1002; explicit GetIterReply(GetIterCommand& cmd); ~GetIterReply() override; @@ -80,7 +81,7 @@ class CreateIteratorCommand : public api::InternalCommand spi::ReadConsistency _readConsistency; public: - static const uint32_t ID = 1003; + static constexpr uint32_t ID = 1003; typedef std::unique_ptr<CreateIteratorCommand> UP; typedef std::shared_ptr<CreateIteratorCommand> SP; @@ -114,7 +115,7 @@ class CreateIteratorReply : public api::InternalReply document::Bucket _bucket; spi::IteratorId _iteratorId; public: - static const uint32_t ID = 1004; + static constexpr uint32_t ID = 1004; typedef std::unique_ptr<CreateIteratorReply> UP; typedef std::shared_ptr<CreateIteratorReply> SP; @@ -132,7 +133,7 @@ class DestroyIteratorCommand : public api::InternalCommand { spi::IteratorId _iteratorId; public: - static const uint32_t ID = 1005; + static constexpr uint32_t ID = 1005; typedef std::unique_ptr<DestroyIteratorCommand> UP; typedef std::shared_ptr<DestroyIteratorCommand> SP; @@ -150,7 +151,7 @@ class DestroyIteratorReply : public api::InternalReply { spi::IteratorId _iteratorId; public: - static const uint32_t ID = 1006; + static constexpr uint32_t ID = 1006; typedef std::unique_ptr<DestroyIteratorReply> UP; typedef std::shared_ptr<DestroyIteratorReply> SP; @@ -164,7 +165,7 @@ class RecheckBucketInfoCommand : public api::InternalCommand { document::Bucket _bucket; public: - static const uint32_t ID = 1007; + static constexpr uint32_t ID = 1007; typedef std::shared_ptr<RecheckBucketInfoCommand> SP; typedef std::unique_ptr<RecheckBucketInfoCommand> UP; @@ -182,7 +183,7 @@ class RecheckBucketInfoReply : public api::InternalReply { document::Bucket _bucket; public: - static const uint32_t ID = 1008; + static constexpr uint32_t ID = 1008; typedef std::shared_ptr<RecheckBucketInfoReply> SP; typedef std::unique_ptr<RecheckBucketInfoReply> UP; @@ -206,7 +207,7 @@ public: } }; - static const uint32_t ID = 1009; + static constexpr uint32_t ID = 1009; typedef std::shared_ptr<AbortBucketOperationsCommand> SP; typedef std::shared_ptr<const AbortBucketOperationsCommand> CSP; private: @@ -227,7 +228,7 @@ public: class AbortBucketOperationsReply : public api::InternalReply { public: - static const uint32_t ID = 1010; + static constexpr uint32_t ID = 1010; typedef std::shared_ptr<AbortBucketOperationsReply> SP; typedef std::shared_ptr<const AbortBucketOperationsReply> CSP; @@ -237,5 +238,36 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; + +// Internal Command task for bringing along a Bucket and a BucketTask in +// the inner workings of the storagelink chain. +class RunTaskCommand : public api::InternalCommand { +public: + static constexpr uint32_t ID = 1011; + RunTaskCommand(const spi::Bucket &bucket, std::unique_ptr<spi::BucketTask> task); + ~RunTaskCommand(); + + document::Bucket getBucket() const override { return _bucket.getBucket(); } + std::unique_ptr<api::StorageReply> makeReply() override; + spi::BucketTask & task() & { + return *_task; + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; +private: + std::unique_ptr<spi::BucketTask> _task; + spi::Bucket _bucket; +}; + +// Simple reply for matching the RunTaskCommand +class RunTaskReply : public api::InternalReply +{ +public: + explicit RunTaskReply(const RunTaskCommand&); + void print(std::ostream& out, bool verbose, const std::string& indent) const override; +private: + static constexpr uint32_t ID = 1012; +}; + } // ns storage diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index cbe5454f4e7..38ffd2c57e7 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -75,6 +75,8 @@ PersistenceHandler::handleCommandSplitByType(api::StorageCommand& msg, MessageTr return _simpleHandler.handleReadBucketInfo(static_cast<ReadBucketInfo&>(msg), std::move(tracker)); case RecheckBucketInfoCommand::ID: return _splitJoinHandler.handleRecheckBucketInfo(static_cast<RecheckBucketInfoCommand&>(msg), std::move(tracker)); + case RunTaskCommand::ID: + return _asyncHandler.handleRunTask(static_cast<RunTaskCommand &>(msg), std::move(tracker)); default: LOG(warning, "Persistence handler received unhandled internal command %s", msg.toString().c_str()); break; diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 99133073a19..b60d3fd8d5d 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -18,7 +18,7 @@ namespace storage::api { } namespace storage::spi { - class PersistenceProvider; + struct PersistenceProvider; } namespace storage { diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.h b/storage/src/vespa/storage/persistence/splitbitdetector.h index d2c415fc526..22261c03f46 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.h +++ b/storage/src/vespa/storage/persistence/splitbitdetector.h @@ -23,7 +23,7 @@ namespace storage { -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } struct SplitBitDetector { diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index f75cf867327..ef835ec9f8a 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -17,7 +17,7 @@ namespace storage { -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } class FileStorManager; diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index 395e33a0393..f2b23724c98 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -49,7 +49,8 @@ StateManager::StateManager(StorageComponentRegister& compReg, _controllers_observed_explicit_node_state(), _noThreadTestMode(testMode), _grabbedExternalLock(false), - _notifyingListeners(false) + _notifyingListeners(false), + _requested_almost_immediate_node_state_replies(false) { _nodeState->setMinUsedBits(58); _nodeState->setStartTimestamp(_component.getClock().getTimeInSeconds().getTime()); @@ -520,21 +521,25 @@ StateManager::run(framework::ThreadHandle& thread) { while (true) { thread.registerTick(); - std::unique_lock guard(_threadLock); - // Take lock before doing stuff, to be sure we don't wait after - // destructor have grabbed lock to stop() us. if (thread.interrupted()) { break; } tick(); - _threadCond.wait_for(guard, 1000ms); + std::unique_lock guard(_threadLock); + if (!_requested_almost_immediate_node_state_replies.load(std::memory_order_relaxed)) { + _threadCond.wait_for(guard, 1000ms); + } } } void StateManager::tick() { - framework::MilliSecTime time(_component.getClock().getTimeInMillis()); + bool almost_immediate_replies = _requested_almost_immediate_node_state_replies.load(std::memory_order_relaxed); + if (almost_immediate_replies) { + _requested_almost_immediate_node_state_replies.store(false, std::memory_order_relaxed); + } + framework::MilliSecTime time(almost_immediate_replies ? framework::MilliSecTime(0) : _component.getClock().getTimeInMillis()); sendGetNodeStateReplies(time); } @@ -637,14 +642,27 @@ StateManager::getNodeInfo() const return json.str(); } +void +StateManager::clear_controllers_observed_explicit_node_state_vector() +{ + std::lock_guard guard(_stateLock); + // Next GetNodeState request from any controller will be replied to instantly + _controllers_observed_explicit_node_state.clear(); +} + void StateManager::immediately_send_get_node_state_replies() { LOG(debug, "Immediately replying to all pending GetNodeState requests"); - { - std::lock_guard guard(_stateLock); - // Next GetNodeState request from any controller will be replied to instantly - _controllers_observed_explicit_node_state.clear(); - } + clear_controllers_observed_explicit_node_state_vector(); sendGetNodeStateReplies(); } +void +StateManager::request_almost_immediate_node_state_replies() +{ + clear_controllers_observed_explicit_node_state_vector(); + std::unique_lock guard(_threadLock); + _requested_almost_immediate_node_state_replies.store(true, std::memory_order_relaxed); + _threadCond.notify_all(); +} + } // storage diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 1731998c14f..9f2853456e5 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -69,6 +69,7 @@ class StateManager : public NodeStateUpdater, bool _noThreadTestMode; bool _grabbedExternalLock; std::atomic<bool> _notifyingListeners; + std::atomic<bool> _requested_almost_immediate_node_state_replies; public: explicit StateManager(StorageComponentRegister&, metrics::MetricManager&, @@ -96,6 +97,7 @@ public: HostInfo& getHostInfo() { return *_hostInfo; } void immediately_send_get_node_state_replies() override; + void request_almost_immediate_node_state_replies() override; private: struct ExternalStateLock; @@ -145,6 +147,8 @@ private: std::string getNodeInfo() const; void run(framework::ThreadHandle&) override; + + void clear_controllers_observed_explicit_node_state_vector(); }; } // storage diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 55e98102d8c..b7f0a8324d6 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -33,7 +33,10 @@ <properties> <vespaversion>${project.version}</vespaversion> - <test-framework.version>${project.version}</test-framework.version> + <!-- when <test-framework.version> is "${project.version}", it is decoupled from the + compile version set with "-D vespaversion=...", and is instead the newest version. + When it is "${vespaversion}", it is coupled, which should be the default. --> + <test-framework.version>${vespaversion}</test-framework.version> <target_jdk_version>11</target_jdk_version> <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> diff --git a/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java b/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java index 8c38cd07a34..28913d94560 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/state/NodeState.java @@ -18,6 +18,8 @@ import java.util.StringTokenizer; */ public class NodeState implements Cloneable { + public static final String ORCHESTRATOR_RESERVED_DESCRIPTION = "Orchestrator"; + private final NodeType type; private State state = State.UP; private String description = ""; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 7b13332052f..884aa954599 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -122,7 +122,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.clock = clock; this.identity = new AthenzService(config.domain(), config.service()); this.ztsEndpoint = URI.create(config.ztsUrl()); - roleSslCertCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::requestRoleCertificate); + roleSslCertCache = crateAutoReloadableCache(ROLE_SSL_CONTEXT_EXPIRY, this::requestRoleCertificate, this.scheduler); roleKeyManagerCache = new HashMap<>(); roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); @@ -145,6 +145,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen }); } + private static <KEY, VALUE> LoadingCache<KEY, VALUE> crateAutoReloadableCache(Duration expiry, Function<KEY, VALUE> cacheLoader, ScheduledExecutorService scheduler) { + LoadingCache<KEY, VALUE> cache = createCache(expiry, cacheLoader); + + // The cache above will reload it's contents if and only if a request for the key is made. Scheduling + // a cache reloader to reload all keys in this cache. + scheduler.scheduleAtFixedRate(() -> { cache.asMap().keySet().forEach(cache::getUnchecked);}, + expiry.dividedBy(4).toMinutes(), + expiry.dividedBy(4).toMinutes(), + TimeUnit.MINUTES); + return cache; + } + private static SSLContext createIdentitySslContext(X509ExtendedKeyManager keyManager, Path trustStore) { return new SslContextBuilder() .withKeyManager(keyManager) diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml index d290f005b5e..7aee1e4265a 100644 --- a/vespa-documentgen-plugin/pom.xml +++ b/vespa-documentgen-plugin/pom.xml @@ -47,6 +47,12 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> <version>3.5.0</version> diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 7119bde7a09..ad10c1af608 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -2,8 +2,8 @@ package ai.vespa.hosted.plugin; import ai.vespa.hosted.api.ControllerHttpClient; -import ai.vespa.hosted.api.Properties; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.InstanceName; import com.yahoo.yolean.Exceptions; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; @@ -16,7 +16,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.stream.Collectors.joining; @@ -56,6 +55,8 @@ public abstract class AbstractVespaMojo extends AbstractMojo { protected ApplicationId id; protected ControllerHttpClient controller; + protected boolean requireInstance() { return false; } + @Override public final void execute() throws MojoExecutionException, MojoFailureException { try { @@ -80,19 +81,24 @@ public abstract class AbstractVespaMojo extends AbstractMojo { /** Return the name of the relevant entity, e.g., application with or without instance. */ protected String name() { return tenant + "." + application; } - protected void setup() { - tenant = firstNonBlank(tenant, project.getProperties().getProperty("tenant")); - application = firstNonBlank(application, project.getProperties().getProperty("application")); - instance = firstNonBlank(instance, project.getProperties().getProperty("instance"), Properties.user()); + protected void setup() throws MojoExecutionException { + tenant = firstNonBlank(tenant, project.getProperties().getProperty("tenant")) + .orElseThrow(() -> new MojoExecutionException("'tenant' must be specified as a parameter or project property")); + application = firstNonBlank(application, project.getProperties().getProperty("application")) + .orElseThrow(() -> new MojoExecutionException("'application' must be specified as a parameter or project property")); + instance = firstNonBlank(instance, project.getProperties().getProperty("instance"), requireInstance() ? null : InstanceName.defaultName().value()) + .orElseThrow(() -> new MojoExecutionException("'instance' must be specified as a parameter or project property")); id = ApplicationId.from(tenant, application, instance); - if (!isNullOrBlank(apiKey)) { + if ( ! isNullOrBlank(apiKey)) { controller = ControllerHttpClient.withSignatureKey(URI.create(endpoint), apiKey, id); - } else if (!isNullOrBlank(apiKeyFile)) { + } + else if ( ! isNullOrBlank(apiKeyFile)) { controller = isNullOrBlank(apiCertificateFile) ? ControllerHttpClient.withSignatureKey(URI.create(endpoint), Paths.get(apiKeyFile), id) : ControllerHttpClient.withKeyAndCertificate(URI.create(endpoint), Paths.get(apiKeyFile), Paths.get(apiCertificateFile)); - } else { + } + else { throw new IllegalArgumentException("One of the properties 'apiKey' or 'apiKeyFile' is required."); } } @@ -102,12 +108,12 @@ public abstract class AbstractVespaMojo extends AbstractMojo { } /** Returns the first of the given strings which is non-null and non-blank, or throws IllegalArgumentException. */ - protected static String firstNonBlank(String... values) { + protected static Optional<String> firstNonBlank(String... values) { for (String value : values) if (value != null && ! value.isBlank()) - return value; + return Optional.of(value); - throw new IllegalArgumentException("No valid value given"); + return Optional.empty(); } protected static Optional<String> optionalOf(String value) { diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java index 03b4dab246f..30a246019e5 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeleteMojo.java @@ -13,6 +13,9 @@ import org.apache.maven.plugins.annotations.Mojo; public class DeleteMojo extends AbstractVespaDeploymentMojo { @Override + protected boolean requireInstance() { return true; } + + @Override protected void doExecute() { if (!isNullOrBlank(environment) && ! Environment.from(environment).isManuallyDeployed()) throw new IllegalArgumentException("Manual deletion is not permitted in " + environment); diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java index 3ca628cdc84..e141261f62b 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java @@ -37,10 +37,14 @@ public class DeployMojo extends AbstractVespaDeploymentMojo { private DeploymentLog.Level loggable; @Override + protected boolean requireInstance() { return true; } + + @Override protected void doExecute() throws MojoFailureException, MojoExecutionException { loggable = DeploymentLog.Level.valueOf(vespaLogLevel); Deployment deployment = Deployment.ofPackage(Paths.get(firstNonBlank(applicationZip, - projectPathOf("target", "application.zip")))); + projectPathOf("target", "application.zip")) + .orElseThrow())); // Fallback always exists. if ( ! isNullOrBlank(vespaVersion)) deployment = deployment.atVersion(vespaVersion); ZoneId zone = zoneOf(environment, region); diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java index 6669f771a0e..96b5ab4244c 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java @@ -43,8 +43,8 @@ public class SubmitMojo extends AbstractVespaMojo { @Override public void doExecute() { - applicationZip = firstNonBlank(applicationZip, projectPathOf("target", "application.zip")); - applicationTestZip = firstNonBlank(applicationTestZip, projectPathOf("target", "application-test.zip")); + applicationZip = firstNonBlank(applicationZip, projectPathOf("target", "application.zip")).orElseThrow(); + applicationTestZip = firstNonBlank(applicationTestZip, projectPathOf("target", "application-test.zip")).orElseThrow(); Submission submission = new Submission(optionalOf(repository), optionalOf(branch), optionalOf(commit), optionalOf(sourceUrl), optionalOf(authorEmail), Paths.get(applicationZip), Paths.get(applicationTestZip), diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SuspendMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SuspendMojo.java index 52057e237d7..40ede218d52 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SuspendMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SuspendMojo.java @@ -16,6 +16,9 @@ public class SuspendMojo extends AbstractVespaDeploymentMojo { private boolean suspend; @Override + protected boolean requireInstance() { return true; } + + @Override protected void doExecute() { getLog().info(controller.suspend(id, zoneOf(environment, region), suspend)); } diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 8254c208588..834c3d7c988 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -78,6 +78,12 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java index c3e42895a5e..36deb318ae8 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java +++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java @@ -11,7 +11,8 @@ public enum CompressionType { // Do not change the type->ordinal association. The gap is due to historic types no longer supported. NONE((byte) 0), INCOMPRESSIBLE((byte) 5), - LZ4((byte) 6); + LZ4((byte) 6), + ZSTD((byte) 7); private byte code; @@ -38,6 +39,8 @@ public enum CompressionType { return INCOMPRESSIBLE; case ((byte) 6): return LZ4; + case ((byte) 7): + return ZSTD; default: throw new IllegalArgumentException("Unknown compression type ordinal " + value); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index fb5da192f36..3220b81a3a9 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -18,6 +18,7 @@ import java.util.Random; */ public class Compressor { + private final ZstdCompressor zstdCompressor = new ZstdCompressor(); private final CompressionType type; private final int level; private final double compressionThresholdFactor; @@ -91,6 +92,11 @@ public class Compressor { if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); return new Compression(CompressionType.LZ4, dataSize, compressedData); + case ZSTD: + int dataLength = uncompressedSize.orElse(data.length); + if (dataLength < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataLength, data); + byte[] compressed = zstdCompressor.compress(data, 0, dataLength); + return new Compression(CompressionType.ZSTD, dataLength, compressed); default: throw new IllegalArgumentException(requestedCompression + " is not supported"); } @@ -130,6 +136,15 @@ public class Compressor { if (expectedCompressedSize.isPresent() && compressedSize != expectedCompressedSize.get()) throw new IllegalStateException("Compressed size mismatch. Expected " + compressedSize + ". Got " + expectedCompressedSize.get()); return uncompressedLZ4Data; + case ZSTD: + int compressedLength = expectedCompressedSize.orElseThrow(() -> new IllegalArgumentException("Zstd decompressor requires input size")); + byte[] decompressedData = zstdCompressor.decompress(compressedData, compressedDataOffset, compressedLength); + expectedCompressedSize.ifPresent(expectedSize -> { + if (compressedData.length != expectedSize) { + throw new IllegalStateException("Compressed size mismatch. Expected " + expectedSize + ". Got " + decompressedData.length); + } + }); + return decompressedData; default: throw new IllegalArgumentException(compression + " is not supported"); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java new file mode 100644 index 00000000000..72ccb730db7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java @@ -0,0 +1,51 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import java.util.Arrays; + +/** + * Frame based Zstd compressor (https://github.com/facebook/zstd) + * Implemented based on https://github.com/airlift/aircompressor - a pure Java implementation (no JNI). + * + * @author bjorncs + */ +public class ZstdCompressor { + + private static final io.airlift.compress.zstd.ZstdCompressor compressor = new io.airlift.compress.zstd.ZstdCompressor(); + private static final io.airlift.compress.zstd.ZstdDecompressor decompressor = new io.airlift.compress.zstd.ZstdDecompressor(); + + public byte[] compress(byte[] input, int inputOffset, int inputLength) { + int maxCompressedLength = getMaxCompressedLength(inputLength); + byte[] output = new byte[maxCompressedLength]; + int compressedLength = compress(input, inputOffset, inputLength, output, 0, maxCompressedLength); + return Arrays.copyOf(output, compressedLength); + } + + public int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) { + return compressor.compress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); + } + + /** + * Note: + * Implementation assumes single frame (since {@link #getDecompressedLength(byte[], int, int)} only includes the first frame) + * The {@link #decompress(byte[], int, int, byte[], int, int)} overload will try to decompress all frame, causing the output buffer to overflow. + */ + public byte[] decompress(byte[] input, int inputOffset, int inputLength) { + int decompressedLength = getDecompressedLength(input, inputOffset, inputLength); + byte[] output = new byte[decompressedLength]; + decompress(input, inputOffset, inputLength, output, 0, decompressedLength); + return output; + } + + public int decompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) { + return decompressor.decompress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); + } + + public static int getMaxCompressedLength(int uncompressedLength) { + return compressor.maxCompressedLength(uncompressedLength); + } + + public static int getDecompressedLength(byte[] input, int inputOffset, int inputLength) { + return (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java new file mode 100644 index 00000000000..e81bcf6a465 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java @@ -0,0 +1,88 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author bjorncs + */ +public class ZstdOuputStream extends OutputStream { + + private final ZstdCompressor compressor = new ZstdCompressor(); + + public static final int DEFAULT_INPUT_BUFFER_SIZE = 8*1024; + + private final OutputStream out; + private final byte[] inputBuffer; + private final byte[] outputBuffer; + private int inputPosition = 0; + private boolean isClosed = false; + + public ZstdOuputStream(OutputStream out, int inputBufferSize) { + this.out = out; + this.inputBuffer = new byte[inputBufferSize]; + this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)]; + } + + public ZstdOuputStream(OutputStream out) { + this(out, DEFAULT_INPUT_BUFFER_SIZE); + } + + @Override + public void write(int b) throws IOException { + throwIfClosed(); + inputBuffer[inputPosition++] = (byte) b; + flushIfFull(); + } + + @Override + public void write(byte[] b) throws IOException { + throwIfClosed(); + write(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throwIfClosed(); + int end = off + len; + while (off < end) { + int copyLength = Math.min(end - off, inputBuffer.length - inputPosition); + System.arraycopy(b, off, inputBuffer, inputPosition, copyLength); + off += copyLength; + inputPosition += copyLength; + flushIfFull(); + } + } + + @Override + public void flush() throws IOException { + flushInternal(); + out.flush(); + } + + @Override + public void close() throws IOException { + throwIfClosed(); + flush(); + out.close(); + isClosed = true; + } + + private void flushInternal() throws IOException { + throwIfClosed(); + int compressedLength = compressor.compress(inputBuffer, 0, inputPosition, outputBuffer, 0, outputBuffer.length); + out.write(outputBuffer, 0, compressedLength); + inputPosition = 0; + } + + private void flushIfFull() throws IOException { + if (inputPosition == inputBuffer.length) { + flushInternal(); + } + } + + private void throwIfClosed() { + if (isClosed) throw new IllegalArgumentException("Output stream is already closed"); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/compress/CompressorTest.java b/vespajlib/src/test/java/com/yahoo/compress/CompressorTest.java new file mode 100644 index 00000000000..0c6af48deb8 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/compress/CompressorTest.java @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bjorncs + */ +class CompressorTest { + + @Test + void compresses_and_decompresses_input_using_zstd() { + byte[] inputData = "The quick brown fox jumps over the lazy dog".getBytes(); + Compressor compressor = new Compressor(CompressionType.ZSTD); + Compressor.Compression compression = compressor.compress(CompressionType.ZSTD, inputData, Optional.empty()); + assertEquals(inputData.length, compression.uncompressedSize()); + byte[] compressedData = compression.data(); + byte[] decompressedData = compressor.decompress(CompressionType.ZSTD, compressedData, 0, inputData.length, Optional.of(compressedData.length)); + assertArrayEquals(inputData, decompressedData); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/compress/ZstdCompressorTest.java b/vespajlib/src/test/java/com/yahoo/compress/ZstdCompressorTest.java new file mode 100644 index 00000000000..f38ce4ad953 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/compress/ZstdCompressorTest.java @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class ZstdCompressorTest { + + @Test + void compresses_and_decompresses_input() { + byte[] inputData = "The quick brown fox jumps over the lazy dog".getBytes(); + ZstdCompressor compressor = new ZstdCompressor(); + byte[] compressedData = compressor.compress(inputData, 0, inputData.length); + byte[] decompressedData = compressor.decompress(compressedData, 0, compressedData.length); + assertArrayEquals(inputData, decompressedData); + } + + @Test + void compressed_size_is_less_than_uncompressed() { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < 100; i++) { + builder.append("The quick brown fox jumps over the lazy dog").append('\n'); + } + byte[] inputData = builder.toString().getBytes(); + ZstdCompressor compressor = new ZstdCompressor(); + byte[] compressedData = compressor.compress(inputData, 0, inputData.length); + assertTrue( + compressedData.length < inputData.length, + () -> "Compressed size is " + compressedData.length + " while uncompressed size is " + inputData.length); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/compress/ZstdOuputStreamTest.java b/vespajlib/src/test/java/com/yahoo/compress/ZstdOuputStreamTest.java new file mode 100644 index 00000000000..5d35eb10215 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/compress/ZstdOuputStreamTest.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class ZstdOuputStreamTest { + + @Test + void output_stream_compresses_input() throws IOException { + byte[] inputData = "The quick brown fox jumps over the lazy dog".getBytes(); + ByteArrayOutputStream arrayOut = new ByteArrayOutputStream(); + try (ZstdOuputStream zstdOut = new ZstdOuputStream(arrayOut, 12)) { + zstdOut.write(inputData[0]); + zstdOut.write(inputData, 1, inputData.length - 1); + } + byte[] compressedData = arrayOut.toByteArray(); + ZstdCompressor compressor = new ZstdCompressor(); + byte[] decompressedData = new byte[inputData.length]; + compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); + assertArrayEquals(inputData, decompressedData); + } + + @Test + void compressed_size_is_less_than_uncompressed() throws IOException { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < 100; i++) { + builder.append("The quick brown fox jumps over the lazy dog").append('\n'); + } + byte[] inputData = builder.toString().getBytes(); + ByteArrayOutputStream arrayOut = new ByteArrayOutputStream(); + try (ZstdOuputStream zstdOut = new ZstdOuputStream(arrayOut)) { + zstdOut.write(inputData); + } + int compressedSize = arrayOut.toByteArray().length; + assertTrue( + compressedSize < inputData.length, + () -> "Compressed size is " + compressedSize + " while uncompressed size is " + inputData.length); + } +} |