diff options
14 files changed, 183 insertions, 55 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index ae6f1fd96e4..6baaea6ea05 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -3,6 +3,7 @@ package com.yahoo.schema; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; +import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collections; import java.util.HashMap; @@ -44,11 +45,37 @@ public class OnnxModel extends DistributableResource { addInputNameMapping(onnxName, vespaName, true); } + private String validateInputSource(String source) { + var optRef = Reference.simple(source); + if (optRef.isPresent()) { + Reference ref = optRef.get(); + // input can be one of: + // attribute(foo), query(foo), constant(foo) + if (FeatureNames.isSimpleFeature(ref)) { + return ref.toString(); + } + // or a function (evaluated by backend) + if (ref.isSimple() && "rankingExpression".equals(ref.name())) { + var arg = ref.simpleArgument(); + if (arg.isPresent()) { + return ref.toString(); + } + } + } else { + // otherwise it must be an identifier + Reference ref = Reference.fromIdentifier(source); + return ref.toString(); + } + // invalid input source + throw new IllegalArgumentException("invalid input for ONNX model " + getName() + ": " + source); + } + public void addInputNameMapping(String onnxName, String vespaName, boolean overwrite) { Objects.requireNonNull(onnxName, "Onnx name cannot be null"); Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + String source = validateInputSource(vespaName); if (overwrite || ! inputMap.containsKey(onnxName)) { - inputMap.put(onnxName, vespaName); + inputMap.put(onnxName, source); } } @@ -59,8 +86,10 @@ public class OnnxModel extends DistributableResource { public void addOutputNameMapping(String onnxName, String vespaName, boolean overwrite) { Objects.requireNonNull(onnxName, "Onnx name cannot be null"); Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + // output name must be a valid identifier: + var ref = Reference.fromIdentifier(vespaName); if (overwrite || ! outputMap.containsKey(onnxName)) { - outputMap.put(onnxName, vespaName); + outputMap.put(onnxName, ref.toString()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 4a4222cca6a..4f262fb8105 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -216,7 +216,7 @@ public class ClusterModel { public Instant at() { return at;} private OptionalDouble cpuCostPerQuery() { - if (averageQueryRate().isEmpty()) return OptionalDouble.empty(); + if (averageQueryRate().isEmpty() || averageQueryRate().getAsDouble() == 0.0) return OptionalDouble.empty(); // TODO: Query rate should generally be sampled at the time where we see the peak resource usage int fanOut = clusterSpec.type().isContainer() ? 1 : groupSize(); return OptionalDouble.of(peakLoad().cpu() * queryCpuFraction() * fanOut * nodes.not().retired().first().get().resources().vcpu() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java index 5cef4baadd4..bd3589be9dd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java @@ -11,6 +11,8 @@ import org.junit.Test; import java.time.Duration; import java.util.Optional; +import static org.junit.Assert.assertEquals; + /** * Tests autoscaling using information from the BCP group this cluster deployment * is part of to supplement local data when the local deployment lacks sufficient traffic. @@ -28,7 +30,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 3.6, 6.1, 25.3, + 9, 1, 3.6, 6.1, 25.3, fixture.autoscale()); // Higher query rate @@ -36,7 +38,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 7.1, 6.1, 25.3, + 9, 1, 7.1, 6.1, 25.3, fixture.autoscale()); // Higher headroom @@ -44,7 +46,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 4.2, 6.1, 25.3, + 9, 1, 4.2, 6.1, 25.3, fixture.autoscale()); // Higher per query cost @@ -52,7 +54,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 5.4, 6.1, 25.3, + 9, 1, 5.4, 6.1, 25.3, fixture.autoscale()); // Bcp elsewhere is 0 - use local only @@ -60,7 +62,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(0, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling using local info", - 8, 1, 1, 7.0, 29.0, + 8, 1, 1, 7.0, 29.0, fixture.autoscale()); } @@ -81,7 +83,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 10.5, 41.0, 168.9, + 3, 3, 10.5, 41.0, 168.9, fixture.autoscale()); // Higher query rate @@ -89,7 +91,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 20.9, 41.0, 168.9, + 3, 3, 20.9, 41.0, 168.9, fixture.autoscale()); // Higher headroom @@ -97,7 +99,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 12.4, 41.0, 168.9, + 3, 3, 12.4, 41.0, 168.9, fixture.autoscale()); // Higher per query cost @@ -105,7 +107,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 15.7, 41.0, 168.9, + 3, 3, 15.7, 41.0, 168.9, fixture.autoscale()); } @@ -123,7 +125,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 16.0, 40.8, + 8, 1, 4.0, 16.0, 40.8, fixture.autoscale()); // Higher query rate (mem and disk changes are due to being assigned larger hosts where we get less overhead share @@ -131,7 +133,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 8.0, 16.0, 40.8, + 8, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher headroom @@ -139,7 +141,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 5, 1, 8.0, 16.0, 40.8, + 5, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher per query cost @@ -147,7 +149,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 6, 1, 8.0, 16.0, 40.8, + 6, 1, 8.0, 16.0, 40.8, fixture.autoscale()); } @@ -160,7 +162,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.3, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 14.2, 7.0, 29.0, + 8, 1, 14.2, 7.0, 29.0, fixture.autoscale()); // Some local traffic @@ -170,7 +172,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration1.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 10.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.9, 7.0, 29.0, + 8, 1, 6.9, 7.0, 29.0, fixture.autoscale()); // Enough local traffic to get half the votes @@ -180,7 +182,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration2.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 50.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.7, 6.1, 25.3, + 9, 1, 2.7, 6.1, 25.3, fixture.autoscale()); // Mostly local @@ -190,7 +192,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration3.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 90.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.1, 6.1, 25.3, + 9, 1, 2.1, 6.1, 25.3, fixture.autoscale()); // Local only @@ -200,7 +202,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration4.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.0, 6.1, 25.3, + 9, 1, 2.0, 6.1, 25.3, fixture.autoscale()); // No group info, should be the same as the above @@ -210,7 +212,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration5.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 2.0, 6.1, 25.3, + 9, 1, 2.0, 6.1, 25.3, fixture.autoscale()); // 40 query rate, no group info (for reference to the below) @@ -220,28 +222,65 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration6.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 1.4, 6.1, 25.3, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); // Local query rate is too low but global is even lower so disregard it, giving the same as above fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.store(new BcpGroupInfo(200/40.0, 1.3, 0.45*40.0)); + fixture.store(new BcpGroupInfo(200 / 40.0, 1.3, 0.45 * 40.0)); Duration duration7 = fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().clock().advance(duration7.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 1.4, 6.1, 25.3, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); // Local query rate is too low to be fully confident, and so is global but as it is slightly larger, incorporate it slightly fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.store(new BcpGroupInfo(200/4.0, 1.3, 0.45*4.0)); + fixture.store(new BcpGroupInfo(200 / 4.0, 1.3, 0.45 * 4.0)); Duration duration8 = fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().clock().advance(duration8.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 1.8, 6.1, 25.3, + 9, 1, 1.8, 6.1, 25.3, fixture.autoscale()); } + /** Tests with varying BCP group info parameters. */ + @Test + public void test_autoscaling_metrics() { + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); + + // Empty has metrics at zero + assertEquals(new Autoscaling.Metrics(0, 0, 0), + fixture.autoscale().metrics()); + + + // No external load mesurements -> 0 + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + assertEquals(new Autoscaling.Metrics(0, 1.0, 0), + fixture.autoscale().metrics()); + + // External load is measured to zero -> 0 + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.loader().addQueryRateMeasurements(10, i -> 0.0); + assertEquals(new Autoscaling.Metrics(0, 1.0, 0), + fixture.autoscale().metrics()); + + // External load + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.loader().addQueryRateMeasurements(10, i -> 110.0); + assertEquals(new Autoscaling.Metrics(110, 1.1, 0.05), + round(fixture.autoscale().metrics())); + } + + private Autoscaling.Metrics round(Autoscaling.Metrics metrics) { + return new Autoscaling.Metrics(Math.round(metrics.queryRate() * 100) / 100.0, + Math.round(metrics.growthRateHeadroom() * 100) / 100.0, + Math.round(metrics.cpuCostPerQuery() * 100) / 100.0); + } + } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/Capability.java b/security-utils/src/main/java/com/yahoo/security/tls/Capability.java index 6a6471aa8ac..dce40681b90 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/Capability.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/Capability.java @@ -30,6 +30,7 @@ public enum Capability implements ToCapabilitySet { CONTENT__METRICS_API("vespa.content.metrics_api"), CONTENT__PROTON_ADMIN_API("vespa.content.proton_admin_api"), CONTENT__SEARCH_API("vespa.content.search_api"), + CONTENT__STATE_API("vespa.content.state_api"), CONTENT__STATUS_PAGES("vespa.content.status_pages"), CONTENT__STORAGE_API("vespa.content.storage_api"), LOGSERVER_API("vespa.logserver.api"), diff --git a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java index 010b8a5b228..197088ff434 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -30,13 +29,14 @@ public class CapabilitySet implements ToCapabilitySet { "vespa.all", Capability.values()); public static final CapabilitySet TELEMETRY = predefined( "vespa.telemetry", - Capability.CONTENT__STATUS_PAGES, Capability.CONTENT__METRICS_API, Capability.CONTAINER__STATE_API, - Capability.METRICSPROXY__METRICS_API, Capability.SENTINEL__CONNECTIVITY_CHECK); + Capability.CONTENT__STATUS_PAGES, Capability.CONTENT__STATE_API, Capability.CONTENT__METRICS_API, + Capability.CONTAINER__STATE_API, Capability.METRICSPROXY__METRICS_API, + Capability.SENTINEL__CONNECTIVITY_CHECK); - private static final CapabilitySet SHARED_CAPABILITIES_APP_NODE = CapabilitySet.of( + private static final CapabilitySet SHARED_CAPABILITIES_APP_NODE = CapabilitySet.unionOf(List.of( Capability.LOGSERVER_API, Capability.CONFIGSERVER__CONFIG_API, Capability.CONFIGSERVER__FILEDISTRIBUTION_API, Capability.CONFIGPROXY__CONFIG_API, - Capability.CONFIGPROXY__FILEDISTRIBUTION_API, Capability.SLOBROK__API, TELEMETRY); + Capability.CONFIGPROXY__FILEDISTRIBUTION_API, Capability.SLOBROK__API, TELEMETRY)); public static final CapabilitySet CONTENT_NODE = predefined( "vespa.content_node", @@ -59,7 +59,7 @@ public class CapabilitySet implements ToCapabilitySet { TELEMETRY); private static CapabilitySet predefined(String name, ToCapabilitySet... capabilities) { - var instance = CapabilitySet.of(capabilities); + var instance = CapabilitySet.unionOf(List.of(capabilities)); PREDEFINED.put(name, instance); return instance; } @@ -85,14 +85,14 @@ public class CapabilitySet implements ToCapabilitySet { return new CapabilitySet(caps); } - public static CapabilitySet unionOf(Collection<CapabilitySet> capSets) { + public static CapabilitySet ofSets(Collection<CapabilitySet> capSets) { EnumSet<Capability> union = EnumSet.noneOf(Capability.class); capSets.forEach(cs -> union.addAll(cs.caps)); return new CapabilitySet(union); } - public static CapabilitySet of(ToCapabilitySet... capabilities) { - return CapabilitySet.unionOf(Arrays.stream(capabilities).map(ToCapabilitySet::toCapabilitySet).toList()); + public static CapabilitySet unionOf(Collection<ToCapabilitySet> caps) { + return CapabilitySet.ofSets(caps.stream().map(ToCapabilitySet::toCapabilitySet).toList()); } public static CapabilitySet of(EnumSet<Capability> caps) { return new CapabilitySet(EnumSet.copyOf(caps)); } @@ -107,8 +107,33 @@ public class CapabilitySet implements ToCapabilitySet { public boolean has(Collection<Capability> caps) { return this.caps.containsAll(caps); } public boolean has(Capability... caps) { return this.caps.containsAll(List.of(caps)); } - public SortedSet<String> toNames() { - return caps.stream().map(Capability::asString).collect(Collectors.toCollection(TreeSet::new)); + public Set<String> toCapabilityNames() { + return caps.stream().map(Capability::asString).collect(Collectors.toSet()); + } + + /** return name of the capability set if predefined, otherwise names of the individual capabilities */ + public Set<String> resolveNames() { + var predefinedName = toPredefinedName().orElse(null); + if (predefinedName != null) return Set.of(predefinedName); + return toCapabilityNames(); + } + + /** @return the name if this is a predefined capability set, or empty if not */ + public Optional<String> toPredefinedName() { + return PREDEFINED.entrySet().stream() + .filter(e -> e.getValue().equals(this)) + .map(Map.Entry::getKey) + .findFirst(); + } + + public static Set<String> resolveNames(Collection<ToCapabilitySet> capabilities) { + var names = new HashSet<String>(); + for (ToCapabilitySet tcs : capabilities) { + if (tcs instanceof Capability c) names.add(c.asString()); + else if (tcs instanceof CapabilitySet cs) names.addAll(cs.resolveNames()); + else throw new IllegalArgumentException(tcs.toString()); + } + return Set.copyOf(names); } public Set<Capability> asSet() { return Collections.unmodifiableSet(caps); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java index d7ea93955af..9252b5619f9 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.security.SubjectAlternativeName.Type.DNS; import static com.yahoo.security.SubjectAlternativeName.Type.URI; @@ -78,10 +79,14 @@ public record ConnectionAuthContext(List<X509Certificate> peerCertificateChain, b.append(". Peer "); if (peer != null) b.append("'").append(peer).append("' "); return b.append("with ").append(peerCertificateString().orElse("<missing-certificate>")).append(". Requires capabilities ") - .append(required.toNames()).append(" but peer has ").append(capabilities.toNames()) + .append(toCapabilityNames(required)).append(" but peer has ").append(toCapabilityNames(capabilities)) .append(".").toString(); } + private static String toCapabilityNames(CapabilitySet capabilities) { + return capabilities.toCapabilityNames().stream().sorted().collect(Collectors.joining(", ", "[", "]")); + } + public Optional<X509Certificate> peerCertificate() { return peerCertificateChain.isEmpty() ? Optional.empty() : Optional.of(peerCertificateChain.get(0)); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java index 746fce0e290..d0e1a33fcac 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java @@ -49,7 +49,7 @@ public class PeerAuthorizer { // TODO Pass this through constructor CapabilityMode capabilityMode = TransportSecurityUtils.getCapabilityMode(); return new ConnectionAuthContext( - certChain, CapabilitySet.unionOf(grantedCapabilities), matchedPolicies, capabilityMode); + certChain, CapabilitySet.ofSets(grantedCapabilities), matchedPolicies, capabilityMode); } private static boolean matchesPolicy(PeerPolicy peerPolicy, String cn, List<String> sans) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java index ea3d4cfe002..f713bcb0b08 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java @@ -1,17 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; +import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; /** * @author bjorncs */ -public record PeerPolicy(String policyName, Optional<String> description, CapabilitySet capabilities, - List<RequiredPeerCredential> requiredCredentials) { +public record PeerPolicy(String policyName, Optional<String> description, Set<String> capabilityNames, + CapabilitySet capabilities, List<RequiredPeerCredential> requiredCredentials) { public PeerPolicy { requiredCredentials = List.copyOf(requiredCredentials); + capabilityNames = Set.copyOf(capabilityNames); + } + + public PeerPolicy(String policyName, Optional<String> description, + CapabilitySet capabilities, List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, capabilities.resolveNames(), capabilities, requiredCredentials); } public PeerPolicy(String policyName, List<RequiredPeerCredential> requiredCredentials) { @@ -21,4 +29,16 @@ public record PeerPolicy(String policyName, Optional<String> description, Capabi public PeerPolicy(String policyName, String description, List<RequiredPeerCredential> requiredCredentials) { this(policyName, Optional.ofNullable(description), CapabilitySet.all(), requiredCredentials); } + + public PeerPolicy(String policyName, Optional<String> description, Collection<ToCapabilitySet> capabilities, + List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, CapabilitySet.resolveNames(capabilities), + CapabilitySet.unionOf(capabilities), requiredCredentials); + } + + public PeerPolicy(String policyName, Optional<String> description, Set<String> capabilities, + List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, capabilities, CapabilitySet.fromNames(capabilities), + requiredCredentials); + } } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java index 34626e23e7a..66b90b32f79 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java @@ -96,15 +96,15 @@ class TransportSecurityOptionsJsonSerializer { throw missingFieldException("required-credentials"); } return new PeerPolicy(authorizedPeer.name, Optional.ofNullable(authorizedPeer.description), - toCapabilities(authorizedPeer.capabilities), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); + toCapabilities(authorizedPeer.capabilities), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); } - private static CapabilitySet toCapabilities(List<String> capabilities) { - if (capabilities == null) return CapabilitySet.all(); + private static Set<String> toCapabilities(List<String> capabilities) { + if (capabilities == null) return Set.of(CapabilitySet.ALL.toPredefinedName().get()); if (capabilities.isEmpty()) throw new IllegalArgumentException("\"capabilities\" array must either be not present " + "(implies all capabilities) or contain at least one capability name"); - return CapabilitySet.fromNames(capabilities); + return Set.copyOf(capabilities); } private static List<RequiredPeerCredential> toRequestPeerCredentials(List<RequiredCredential> requiredCredentials) { @@ -148,7 +148,7 @@ class TransportSecurityOptionsJsonSerializer { authorizedPeer.description = peerPolicy.description().orElse(null); CapabilitySet caps = peerPolicy.capabilities(); if (!caps.hasAll()) { - authorizedPeer.capabilities = List.copyOf(caps.toNames()); + authorizedPeer.capabilities = peerPolicy.capabilityNames().stream().sorted().toList(); } for (RequiredPeerCredential requiredPeerCredential : peerPolicy.requiredCredentials()) { RequiredCredential requiredCredential = new RequiredCredential(); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java b/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java index 87b16dbff1f..3fa75df27e1 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java @@ -4,8 +4,6 @@ package com.yahoo.security.tls; import org.junit.jupiter.api.Test; import java.util.Arrays; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -17,10 +15,10 @@ class CapabilitySetTest { @Test void contains_all_capabilities() { - SortedSet<String> expectedNames = Arrays.stream(Capability.values()) + var expectedNames = Arrays.stream(Capability.values()) .map(Capability::asString) - .collect(Collectors.toCollection(TreeSet::new)); - SortedSet<String> actualNames = CapabilitySet.all().toNames(); + .collect(Collectors.toSet()); + var actualNames = CapabilitySet.all().toCapabilityNames(); assertEquals(expectedNames, actualNames); } diff --git a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp index 0bf04289a65..34eefe9c435 100644 --- a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp +++ b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp @@ -84,6 +84,7 @@ TEST("All known capabilities can be looked up by name, and resolve back to same check_capability_mapping("vespa.content.metrics_api", Capability::content_metrics_api()); check_capability_mapping("vespa.content.proton_admin_api", Capability::content_proton_admin_api()); check_capability_mapping("vespa.content.search_api", Capability::content_search_api()); + check_capability_mapping("vespa.content.state_api", Capability::content_state_api()); check_capability_mapping("vespa.content.status_pages", Capability::content_status_pages()); check_capability_mapping("vespa.content.storage_api", Capability::content_storage_api()); check_capability_mapping("vespa.logserver.api", Capability::logserver_api()); @@ -109,6 +110,7 @@ TEST("CapabilitySet instances can be stringified") { "vespa.container.state_api, " "vespa.content.document_api, " "vespa.content.metrics_api, " + "vespa.content.state_api, " "vespa.content.status_pages, " "vespa.content.storage_api, " "vespa.logserver.api, " @@ -135,7 +137,7 @@ TEST("Resolving a capability set adds all its underlying capabilities") { CapabilitySet caps; EXPECT_TRUE(caps.resolve_and_add("vespa.content_node")); // Slightly suboptimal; this test will fail if the default set of capabilities for vespa.content_node changes. - EXPECT_EQUAL(caps.count(), 14u); + EXPECT_EQUAL(caps.count(), 15u); EXPECT_FALSE(caps.empty()); EXPECT_TRUE(caps.contains(Capability::content_storage_api())); EXPECT_TRUE(caps.contains(Capability::content_document_api())); @@ -147,6 +149,7 @@ TEST("Resolving a capability set adds all its underlying capabilities") { EXPECT_TRUE(caps.contains(Capability::configproxy_config_api())); EXPECT_TRUE(caps.contains(Capability::configproxy_filedistribution_api())); // vespa.content_node -> shared node caps -> vespa.telemetry + EXPECT_TRUE(caps.contains(Capability::content_state_api())); EXPECT_TRUE(caps.contains(Capability::content_status_pages())); EXPECT_TRUE(caps.contains(Capability::content_metrics_api())); EXPECT_TRUE(caps.contains(Capability::container_state_api())); diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.cpp b/vespalib/src/vespa/vespalib/net/tls/capability.cpp index cfc1cc7a7cc..49f8aa11bad 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/capability.cpp @@ -35,6 +35,7 @@ constexpr std::array<std::string_view, Capability::max_value_count()> capability "vespa.content.metrics_api"sv, "vespa.content.proton_admin_api"sv, "vespa.content.search_api"sv, + "vespa.content.state_api"sv, "vespa.content.status_pages"sv, "vespa.content.storage_api"sv, "vespa.logserver.api"sv, @@ -83,6 +84,7 @@ std::optional<Capability> Capability::find_capability(const string& cap_name) no {"vespa.content.metrics_api", content_metrics_api()}, {"vespa.content.proton_admin_api", content_proton_admin_api()}, {"vespa.content.search_api", content_search_api()}, + {"vespa.content.state_api", content_state_api()}, {"vespa.content.status_pages", content_status_pages()}, {"vespa.content.storage_api", content_storage_api()}, {"vespa.logserver.api", logserver_api()}, diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.h b/vespalib/src/vespa/vespalib/net/tls/capability.h index a7a1dcd15ac..396fad4cbcd 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability.h +++ b/vespalib/src/vespa/vespalib/net/tls/capability.h @@ -47,6 +47,7 @@ private: ContentMetricsApi, ContentProtonAdminApi, ContentSearchApi, + ContentStateApi, ContentStatusPages, ContentStorageApi, LogserverApi, @@ -176,6 +177,10 @@ public: return Capability(Id::ContentSearchApi); } + constexpr static Capability content_state_api() noexcept { + return Capability(Id::ContentStateApi); + } + constexpr static Capability content_proton_admin_api() noexcept { return Capability(Id::ContentProtonAdminApi); } diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp index 1b879f0c635..e457bcf6cb7 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp @@ -72,6 +72,7 @@ CapabilitySet CapabilitySet::container_node() noexcept { CapabilitySet CapabilitySet::telemetry() noexcept { return CapabilitySet::of({Capability::content_status_pages(), + Capability::content_state_api(), Capability::content_metrics_api(), Capability::container_state_api(), Capability::metricsproxy_metrics_api(), |