diff options
31 files changed, 965 insertions, 59 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index 8fa8bdb66bf..5c2ac402dfa 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -77,7 +77,7 @@ public class FS4InvokerFactory { if (node.isWorking()) { Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); if (backend.probeConnection()) { - invokers.add(node.key(), new FS4SearchInvoker(searcher, query, backend.openChannel(), Optional.of(node))); + invokers.add(new FS4SearchInvoker(searcher, query, backend.openChannel(), Optional.of(node))); nodeAdded = true; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java index da32cfc4fda..eb9ab75c0d0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java @@ -59,8 +59,6 @@ public class FS4SearchInvoker extends SearchInvoker implements ResponseMonitor<F this.query = query; this.queryPacket = queryPacket; - channel.setResponseMonitor(this); - try { boolean couldSend = channel.sendPacket(queryPacket); if (!couldSend) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index a9ae0c34028..8710b47b914 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -15,6 +15,7 @@ import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.hosted.provision.flag.Flags; import com.yahoo.vespa.hosted.provision.maintenance.PeriodicApplicationMaintainer; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.NodeAcl; @@ -80,6 +81,7 @@ public class NodeRepository extends AbstractComponent { private final NameResolver nameResolver; private final DockerImage dockerImage; private final OsVersions osVersions; + private final Flags flags; /** * Creates a node repository from a zookeeper provider. @@ -103,6 +105,7 @@ public class NodeRepository extends AbstractComponent { this.nameResolver = nameResolver; this.dockerImage = dockerImage; this.osVersions = new OsVersions(this.db); + this.flags = new Flags(this.db); // read and write all nodes to make sure they are stored in the latest version of the serialized format for (Node.State state : Node.State.values()) @@ -121,6 +124,11 @@ public class NodeRepository extends AbstractComponent { /** Returns the OS versions to use for nodes in this */ public OsVersions osVersions() { return osVersions; } + /** Returns feature flags of this node repository */ + public Flags flags() { + return flags; + } + // ---------------- Query API ---------------------------------------------------------------- /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java new file mode 100644 index 00000000000..a22e1dea024 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java @@ -0,0 +1,109 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.flag; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Represents a feature flag and its status. Use {@link Flags#get(FlagId)} to lookup status for a specific flag. + * + * @author mpolden + */ +public class Flag { + + private final FlagId id; + private final boolean enabled; + private final Set<String> hostnames; + private final Set<ApplicationId> applications; + + public Flag(FlagId id, boolean enabled, Set<String> hostnames, Set<ApplicationId> applications) { + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.enabled = enabled; + this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); + this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications, "applications must be non-null")); + } + + public FlagId id() { + return id; + } + + /** The hostnames this flag should apply to */ + public Set<String> hostnames() { + return hostnames; + } + + /** The applications this flag should apply to */ + public Set<ApplicationId> applications() { + return applications; + } + + /** + * Returns whether this flag is enabled for all dimensions. Note: More specific dimensions always return true when + * this is true + */ + public boolean isEnabled() { + return enabled; + } + + /** Returns whether this flag is enabled for given hostname */ + public boolean isEnabled(HostName hostname) { + return enabled || hostnames.contains(hostname.value()); + } + + /** Returns whether this flag is enabled for given application */ + public boolean isEnabled(ApplicationId application) { + return enabled || applications.contains(application); + } + + /** Returns a copy of this with this flag enabled for all dimensions */ + public Flag withEnabled(boolean enabled) { + return new Flag(id, enabled, hostnames, applications); + } + + /** Returns a copy of this with enabled set for hostname */ + public Flag withEnabled(HostName hostname, boolean enabled) { + Set<String> hostnames = new LinkedHashSet<>(this.hostnames); + if (enabled) { + hostnames.add(hostname.value()); + } else { + hostnames.remove(hostname.value()); + } + return new Flag(id, this.enabled, hostnames, applications); + } + + /** Returns a copy of this with enabled set for application */ + public Flag withEnabled(ApplicationId application, boolean enabled) { + Set<ApplicationId> applications = new LinkedHashSet<>(this.applications); + if (enabled) { + applications.add(application); + } else { + applications.remove(application); + } + return new Flag(id, this.enabled, hostnames, applications); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Flag flag = (Flag) o; + return id == flag.id; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + /** Create a flag for given feature that is disabled for all dimensions */ + public static Flag disabled(FlagId id) { + return new Flag(id, false, Collections.emptySet(), Collections.emptySet()); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java new file mode 100644 index 00000000000..1b798c14588 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.flag; + +import java.util.Arrays; + +/** + * Features of this node repository that can be toggled. + * + * @author mpolden + */ +public enum FlagId { + + /** Indicates whether a exclusive load balancer should be provisioned */ + exclusiveLoadBalancer("exclusive-load-balancer"); + + private final String serializedValue; + + FlagId(String serializedValue) { + this.serializedValue = serializedValue; + } + + public String serializedValue() { + return serializedValue; + } + + public static FlagId fromSerializedForm(String value) { + return Arrays.stream(FlagId.values()) + .filter(f -> f.serializedValue().equals(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Could not find flag ID by serialized value '" + + value + "'")); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java new file mode 100644 index 00000000000..b4ecf415ede --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java @@ -0,0 +1,70 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.flag; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * This class provides feature flags for the node repository. A feature flag can be toggled for the following + * dimensions: + * + * 1) The node repository (entire zone) + * 2) A specific node + * 3) A specific application + * + * Code which needs to consider feature flags can access them through {@link NodeRepository#flags()}. + * + * @author mpolden + */ +public class Flags { + + private final CuratorDatabaseClient db; + + public Flags(CuratorDatabaseClient db) { + this.db = Objects.requireNonNull(db, "db must be non-null"); + } + + /** Get status for given feature flag */ + public Flag get(FlagId id) { + return db.readFlag(id).orElseGet(() -> Flag.disabled(id)); + } + + /** Get all known feature flags */ + public List<Flag> list() { + return Arrays.stream(FlagId.values()) + .map(this::get) + .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + } + + /** Enable feature flag in this node repository */ + public void setEnabled(FlagId flag, boolean enabled) { + write(flag, (f) -> f.withEnabled(enabled)); + } + + /** Enable feature flag for given application */ + public void setEnabled(FlagId flag, ApplicationId application, boolean enabled) { + write(flag, (f) -> f.withEnabled(application, enabled)); + } + + /** Enable feature flag for given node */ + public void setEnabled(FlagId flag, HostName hostname, boolean enabled) { + write(flag, (f) -> f.withEnabled(hostname, enabled)); + } + + private void write(FlagId id, Function<Flag, Flag> updateFunc) { + try (Lock lock = db.lockFlags()) { + db.writeFlag(updateFunc.apply(get(id))); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index d824f9fa53b..0fc5626241f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -16,6 +16,8 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.flag.Flag; +import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -55,6 +57,7 @@ public class CuratorDatabaseClient { private static final Path root = Path.fromString("/provision/v1"); private static final Path lockRoot = root.append("locks"); private static final Path loadBalancersRoot = root.append("loadBalancers"); + private static final Path flagsRoot = root.append("flags"); private static final Duration defaultLockTimeout = Duration.ofMinutes(2); private final NodeSerializer nodeSerializer; @@ -453,4 +456,26 @@ public class CuratorDatabaseClient { return loadBalancersRoot.append(id.serializedForm()); } + public void writeFlag(Flag flag) { + Path path = flagPath(flag.id()); + curatorDatabase.create(path); + NestedTransaction transaction = new NestedTransaction(); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.setData(path.getAbsolute(), + FlagSerializer.toJson(flag))); + transaction.commit(); + } + + public Optional<Flag> readFlag(FlagId id) { + return read(flagPath(id), FlagSerializer::fromJson); + } + + public Lock lockFlags() { + return lock(lockRoot.append("flagsLock"), defaultLockTimeout); + } + + private Path flagPath(FlagId id) { + return flagsRoot.append(id.serializedValue()); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java new file mode 100644 index 00000000000..431aa92a513 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java @@ -0,0 +1,65 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.provision.flag.Flag; +import com.yahoo.vespa.hosted.provision.flag.FlagId; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * @author mpolden + */ +public class FlagSerializer { + + private static final String featureField = "feature"; + private static final String enabledField = "enabled"; + private static final String hostnamesField = "hostnames"; + private static final String applicationsField = "applications"; + + public static byte[] toJson(Flag flag) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + root.setString(featureField, flag.id().serializedValue()); + root.setBool(enabledField, flag.isEnabled()); + + Cursor nodeArray = root.setArray(hostnamesField); + flag.hostnames().forEach(nodeArray::addString); + + Cursor applicationArray = root.setArray(applicationsField); + flag.applications().forEach(application -> applicationArray.addString(application.serializedForm())); + + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static Flag fromJson(byte[] data) { + Inspector inspect = SlimeUtils.jsonToSlime(data).get(); + + Set<String> hostnames = new LinkedHashSet<>(); + inspect.field(hostnamesField).traverse((ArrayTraverser) (i, hostname) -> hostnames.add(hostname.asString())); + + Set<ApplicationId> applications = new LinkedHashSet<>(); + inspect.field(applicationsField).traverse((ArrayTraverser) (i, application) -> { + applications.add(ApplicationId.fromSerializedForm(application.asString())); + }); + + return new Flag(FlagId.fromSerializedForm(inspect.field(featureField).asString()), + inspect.field(enabledField).asBool(), + hostnames, + applications); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index f8067cb8661..09f8fc41239 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -19,6 +19,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -104,6 +105,9 @@ public class NodeRepositoryProvisioner implements Provisioner { public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { validate(hosts); activator.activate(application, hosts, transaction); + if (nodeRepository.flags().get(FlagId.exclusiveLoadBalancer).isEnabled(application)) { + // TODO: Provision load balancer + } } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java new file mode 100644 index 00000000000..1bc016bcda2 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java @@ -0,0 +1,51 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi.v2; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.flag.Flag; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * @author mpolden + */ +public class FlagsResponse extends HttpResponse { + + private final List<Flag> flags; + + public FlagsResponse(List<Flag> flags) { + super(200); + this.flags = flags; + } + + @Override + public void render(OutputStream out) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor flagArray = root.setArray("flags"); + flags.forEach(flag -> { + Cursor flagObject = flagArray.addObject(); + flagObject.setString("id", flag.id().serializedValue()); + flagObject.setBool("enabled", flag.isEnabled()); + Cursor nodeArray = flagObject.setArray("enabledHostnames"); + flag.hostnames().forEach(nodeArray::addString); + Cursor applicationArray = flagObject.setArray("enabledApplications"); + flag.applications().stream() + .map(ApplicationId::serializedForm) + .forEach(applicationArray::addString); + }); + new JsonFormat(true).encode(out, slime); + } + + @Override + public String getContentType() { + return "application/json"; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index df6bd7d0f15..418a3b16e2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -2,13 +2,16 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.restapi.Path; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -16,6 +19,7 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -103,6 +107,7 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl()); if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions()); + if (path.equals( "/nodes/v2/flags/")) return new FlagsResponse(nodeRepository.flags().list()); throw new NotFoundException("Nothing at path '" + path + "'"); } @@ -149,40 +154,38 @@ public class NodesApiHandler extends LoggingRequestHandler { } private HttpResponse handlePOST(HttpRequest request) { - String path = request.getUri().getPath(); - if (path.equals("/nodes/v2/command/restart")) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/nodes/v2/command/restart")) { int restartCount = nodeRepository.restart(toNodeFilter(request)).size(); return new MessageResponse("Scheduled restart of " + restartCount + " matching nodes"); } - else if (path.equals("/nodes/v2/command/reboot")) { + if (path.matches("/nodes/v2/command/reboot")) { int rebootCount = nodeRepository.reboot(toNodeFilter(request)).size(); return new MessageResponse("Scheduled reboot of " + rebootCount + " matching nodes"); } - else if (path.equals("/nodes/v2/node")) { + if (path.matches("/nodes/v2/node")) { int addedNodes = addNodes(request.getData()); return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state"); } - else if (path.startsWith("/nodes/v2/maintenance/inactive/")) { - return setActive(lastElement(path), false); - } - else { - throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); - } + if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false); + if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), true, "", ""); + if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), true, path.get("dimension"), path.get("value")); + + throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } private HttpResponse handleDELETE(HttpRequest request) { - String path = request.getUri().getPath(); - if (path.startsWith("/nodes/v2/node/")) { - String hostname = lastElement(path); + Path path = new Path(request.getUri().getPath()); + if (path.matches("/nodes/v2/node/{hostname}")) { + String hostname = path.get("hostname"); List<Node> removedNodes = nodeRepository.removeRecursively(hostname); return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); } - else if (path.startsWith("/nodes/v2/maintenance/inactive/")) { - return setActive(lastElement(path), true); - } - else { - throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); - } + if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true); + if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), false, "", ""); + if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), false, path.get("dimension"), path.get("value")); + + throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } private Node nodeFromRequest(HttpRequest request) { @@ -268,13 +271,31 @@ public class NodesApiHandler extends LoggingRequestHandler { return false; } - private MessageResponse setActive(String jobName, boolean active) { + private MessageResponse setJobActive(String jobName, boolean active) { if ( ! maintenance.jobControl().jobs().contains(jobName)) throw new NotFoundException("No job named '" + jobName + "'"); maintenance.jobControl().setActive(jobName, active); return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); } + private HttpResponse setFlag(String flag, boolean enabled, String dimension, String value) { + FlagId flagId = FlagId.fromSerializedForm(flag); + switch (dimension) { + case "application": + nodeRepository.flags().setEnabled(flagId, ApplicationId.fromSerializedForm(value), enabled); + break; + case "node": + nodeRepository.flags().setEnabled(flagId, HostName.from(value), enabled); + break; + case "": + nodeRepository.flags().setEnabled(flagId, enabled); + break; + default: throw new IllegalArgumentException("Unknown flag dimension '" + dimension + "'"); + } + return new MessageResponse((enabled ? "Enabled" : "Disabled") + " feature " + flagId + + (!value.isEmpty() ? " for " + dimension + " '" + value + "'" : "")); + } + private MessageResponse setTargetVersions(HttpRequest request) { NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); Inspector inspector = toSlime(request.getData()).get(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java new file mode 100644 index 00000000000..5018b18c491 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java @@ -0,0 +1,64 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.flag; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; +import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; +import org.junit.Test; + +import java.util.function.Supplier; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author mpolden + */ +public class FlagsTest { + + @Test + public void test_flag_toggling() { + NodeRepository nodeRepository = new MockNodeRepository(new MockCurator(), new MockNodeFlavors()); + Flags flags = nodeRepository.flags(); + Supplier<Flag> flag = () -> flags.get(FlagId.exclusiveLoadBalancer); + + // Flag is disabled by default + assertFalse(flag.get().isEnabled()); + + // Toggle flag for a node + { + HostName node1 = HostName.from("host1"); + flags.setEnabled(FlagId.exclusiveLoadBalancer, node1, true); + assertTrue(flag.get().isEnabled(node1)); + assertFalse(flag.get().isEnabled()); + flags.setEnabled(FlagId.exclusiveLoadBalancer, node1, false); + assertFalse(flag.get().isEnabled(node1)); + } + + // Toggle flag for an application + { + ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); + flags.setEnabled(FlagId.exclusiveLoadBalancer, app1, true); + assertTrue(flag.get().isEnabled(app1)); + assertFalse(flag.get().isEnabled()); + flags.setEnabled(FlagId.exclusiveLoadBalancer, app1, false); + assertFalse(flag.get().isEnabled(app1)); + } + + // Toggle flag globally + { + flags.setEnabled(FlagId.exclusiveLoadBalancer, true); + assertTrue(flag.get().isEnabled()); + // Flag is implicitly enabled for all dimensions + assertTrue(flag.get().isEnabled(HostName.from("host1"))); + assertTrue(flag.get().isEnabled(ApplicationId.from("tenant1", "application1", "default"))); + flags.setEnabled(FlagId.exclusiveLoadBalancer, false); + assertFalse(flag.get().isEnabled()); + } + } + + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java new file mode 100644 index 00000000000..15f2289d340 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.provision.flag.Flag; +import com.yahoo.vespa.hosted.provision.flag.FlagId; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class FlagSerializerTest { + + @Test + public void test_serialization() { + Flag flag = new Flag(FlagId.exclusiveLoadBalancer, true, + ImmutableSet.of("host1", "host2"), + Collections.singleton( + ApplicationId.from("tenant1", "application1", "default") + )); + Flag serialized = FlagSerializer.fromJson(FlagSerializer.toJson(flag)); + assertEquals(flag.id(), serialized.id()); + assertEquals(flag.isEnabled(), serialized.isEnabled()); + assertEquals(flag.hostnames(), serialized.hostnames()); + assertEquals(flag.applications(), serialized.applications()); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 2ff1e403e35..ae7f3f14975 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -659,6 +659,44 @@ public class RestApiTest { "]}"); } + @Test + public void test_flags() throws Exception { + assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags1.json"); + + // Enable flag for application + assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/application/foo:bar:default", + new byte[0], Request.Method.POST), + "{\"message\":\"Enabled feature exclusiveLoadBalancer for application 'foo:bar:default'\"}"); + + // Enable flag for node + assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/node/host1", + new byte[0], Request.Method.POST), + "{\"message\":\"Enabled feature exclusiveLoadBalancer for node 'host1'\"}"); + + assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags2.json"); + + // Enable flag for entire repository + assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer", + new byte[0], Request.Method.POST), + "{\"message\":\"Enabled feature exclusiveLoadBalancer\"}"); + + // Disable flag for application + assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/application/foo:bar:default", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Disabled feature exclusiveLoadBalancer for application 'foo:bar:default'\"}"); + + // Disable flag for node + assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/node/host1", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Disabled feature exclusiveLoadBalancer for node 'host1'\"}"); + + // Disable flag for entire repository + assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Disabled feature exclusiveLoadBalancer\"}"); + + } + /** Tests the rendering of each node separately to make it easier to find errors */ @Test public void test_single_node_rendering() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json new file mode 100644 index 00000000000..8fd09b4a274 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json @@ -0,0 +1,10 @@ +{ + "flags": [ + { + "id": "exclusive-load-balancer", + "enabled": false, + "enabledHostnames": [], + "enabledApplications": [] + } + ] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json new file mode 100644 index 00000000000..78de52e4e85 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json @@ -0,0 +1,14 @@ +{ + "flags": [ + { + "id": "exclusive-load-balancer", + "enabled": false, + "enabledHostnames": [ + "host1" + ], + "enabledApplications": [ + "foo:bar:default" + ] + } + ] +} diff --git a/parent/pom.xml b/parent/pom.xml index 590e89668f0..c60ed3e8d4d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -688,6 +688,11 @@ <artifactId>tensorflow</artifactId> <version>${tensorflow.version}</version> </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <version>3.11.1</version> + </dependency> </dependencies> </dependencyManagement> diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index c1373a391f0..4d9d48c4926 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -948,7 +948,6 @@ void ConformanceTest::testUpdate() { CPPUNIT_ASSERT(!result.hasDocument()); } - { UpdateResult result = spi->update(bucket, Timestamp(6), update, context); @@ -957,6 +956,32 @@ void ConformanceTest::testUpdate() { CPPUNIT_ASSERT_EQUAL(Result::NONE, result.getErrorCode()); CPPUNIT_ASSERT_EQUAL(Timestamp(0), result.getExistingTimestamp()); } + + { + GetResult result = spi->get(bucket, document::AllFields(), doc1->getId(), context); + CPPUNIT_ASSERT_EQUAL(Result::NONE, result.getErrorCode()); + CPPUNIT_ASSERT_EQUAL(Timestamp(0), result.getTimestamp()); + CPPUNIT_ASSERT(!result.hasDocument()); + } + + update->setCreateIfNonExistent(true); + { + // Document does not exist (and therefore its condition cannot match by definition), + // but since CreateIfNonExistent is set it should be auto-created anyway. + UpdateResult result = spi->update(bucket, Timestamp(7), update, context); + spi->flush(bucket, context); + CPPUNIT_ASSERT_EQUAL(Result::NONE, result.getErrorCode()); + CPPUNIT_ASSERT_EQUAL(Timestamp(7), result.getExistingTimestamp()); + } + + { + GetResult result = spi->get(bucket, document::AllFields(), doc1->getId(), context); + CPPUNIT_ASSERT_EQUAL(Result::NONE, result.getErrorCode()); + CPPUNIT_ASSERT_EQUAL(Timestamp(7), result.getTimestamp()); + CPPUNIT_ASSERT_EQUAL(document::IntFieldValue(42), + reinterpret_cast<document::IntFieldValue&>( + *result.getDocument().getValue("headerval"))); + } } void ConformanceTest::testGet() { diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp index 5e6e908e042..7f6bd835cd5 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp @@ -1,9 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "abstractpersistenceprovider.h" +#include <vespa/document/datatype/documenttype.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldset/fieldsets.h> - +#include <vespa/document/fieldvalue/document.h> namespace storage { @@ -19,20 +20,27 @@ AbstractPersistenceProvider::update(const Bucket& bucket, Timestamp ts, return UpdateResult(getResult.getErrorCode(), getResult.getErrorMessage()); } - if (!getResult.hasDocument()) { - return UpdateResult(); + auto docToUpdate = getResult.getDocumentPtr(); + Timestamp updatedTs = getResult.getTimestamp(); + if (!docToUpdate) { + if (!upd->getCreateIfNonExistent()) { + return UpdateResult(); + } else { + docToUpdate = std::make_shared<document::Document>(upd->getType(), upd->getId()); + updatedTs = ts; + } } - upd->applyTo(getResult.getDocument()); + upd->applyTo(*docToUpdate); - Result putResult = put(bucket, ts, getResult.getDocumentPtr(), context); + Result putResult = put(bucket, ts, docToUpdate, context); if (putResult.hasError()) { return UpdateResult(putResult.getErrorCode(), putResult.getErrorMessage()); } - return UpdateResult(getResult.getTimestamp()); + return UpdateResult(updatedTs); } RemoveResult diff --git a/security-utils/pom.xml b/security-utils/pom.xml index 6f094f28362..450428c5c14 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -49,6 +49,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/AuthorizationResult.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/AuthorizationResult.java new file mode 100644 index 00000000000..bcc2fa0e698 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/AuthorizationResult.java @@ -0,0 +1,55 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls.authz; + +import com.yahoo.security.tls.policy.Role; + +import java.util.Collections; +import java.util.Objects; +import java.util.Set; + +/** + * @author bjorncs + */ +public class AuthorizationResult { + private final Set<Role> assumedRoles; + private final Set<String> matchedPolicies; + + public AuthorizationResult(Set<Role> assumedRoles, Set<String> matchedPolicies) { + this.assumedRoles = Collections.unmodifiableSet(assumedRoles); + this.matchedPolicies = Collections.unmodifiableSet(matchedPolicies); + } + + public Set<Role> assumedRoles() { + return assumedRoles; + } + + public Set<String> matchedPolicies() { + return matchedPolicies; + } + + public boolean succeeded() { + return matchedPolicies.size() > 0; + } + + @Override + public String toString() { + return "AuthorizationResult{" + + "assumedRoles=" + assumedRoles + + ", matchedPolicies=" + matchedPolicies + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AuthorizationResult that = (AuthorizationResult) o; + return Objects.equals(assumedRoles, that.assumedRoles) && + Objects.equals(matchedPolicies, that.matchedPolicies); + } + + @Override + public int hashCode() { + return Objects.hash(assumedRoles, matchedPolicies); + } +} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizer.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizer.java new file mode 100644 index 00000000000..bead32fe309 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizer.java @@ -0,0 +1,75 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls.authz; + +import com.yahoo.security.SubjectAlternativeName; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.policy.AuthorizedPeers; +import com.yahoo.security.tls.policy.PeerPolicy; +import com.yahoo.security.tls.policy.RequiredPeerCredential; +import com.yahoo.security.tls.policy.Role; + +import java.security.cert.X509Certificate; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.yahoo.security.SubjectAlternativeName.Type.DNS_NAME; +import static com.yahoo.security.SubjectAlternativeName.Type.IP_ADDRESS; +import static java.util.stream.Collectors.toList; + +/** + * Uses rules from {@link AuthorizedPeers} to evaluate X509 certificates + * + * @author bjorncs + */ +public class PeerAuthorizer { + private final AuthorizedPeers authorizedPeers; + + public PeerAuthorizer(AuthorizedPeers authorizedPeers) { + this.authorizedPeers = authorizedPeers; + } + + public AuthorizationResult authorizePeer(X509Certificate peerCertificate) { + Set<Role> assumedRoles = new HashSet<>(); + Set<String> matchedPolicies = new HashSet<>(); + String cn = getCommonName(peerCertificate).orElse(null); + List<String> sans = getSubjectAlternativeNames(peerCertificate); + for (PeerPolicy peerPolicy : authorizedPeers.peerPolicies()) { + if (matchesPolicy(peerPolicy, cn, sans)) { + assumedRoles.addAll(peerPolicy.assumedRoles()); + matchedPolicies.add(peerPolicy.policyName()); + } + } + return new AuthorizationResult(assumedRoles, matchedPolicies); + } + + private static boolean matchesPolicy(PeerPolicy peerPolicy, String cn, List<String> sans) { + return peerPolicy.requiredCredentials().stream() + .allMatch(requiredCredential -> matchesRequiredCredentials(requiredCredential, cn, sans)); + } + + private static boolean matchesRequiredCredentials(RequiredPeerCredential requiredCredential, String cn, List<String> sans) { + switch (requiredCredential.field()) { + case CN: + return cn != null && requiredCredential.pattern().matches(cn); + case SAN_DNS: + return sans.stream() + .anyMatch(san -> requiredCredential.pattern().matches(san)); + default: + throw new RuntimeException("Unknown field: " + requiredCredential.field()); + } + } + + private static Optional<String> getCommonName(X509Certificate peerCertificate) { + return X509CertificateUtils.getSubjectCommonNames(peerCertificate).stream() + .findFirst(); + } + + private static List<String> getSubjectAlternativeNames(X509Certificate peerCertificate) { + return X509CertificateUtils.getSubjectAlternativeNames(peerCertificate).stream() + .filter(san -> san.getType() == DNS_NAME || san.getType() == IP_ADDRESS) + .map(SubjectAlternativeName::getValue) + .collect(toList()); + } +} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/package-info.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/package-info.java new file mode 100644 index 00000000000..1379e090d08 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.security.tls.authz; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/security-utils/src/test/java/com/yahoo/security/tls/authz/PeerAuthorizerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/authz/PeerAuthorizerTest.java new file mode 100644 index 00000000000..ffda4fe3c2b --- /dev/null +++ b/security-utils/src/test/java/com/yahoo/security/tls/authz/PeerAuthorizerTest.java @@ -0,0 +1,138 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls.authz; + +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.tls.policy.AuthorizedPeers; +import com.yahoo.security.tls.policy.HostGlobPattern; +import com.yahoo.security.tls.policy.PeerPolicy; +import com.yahoo.security.tls.policy.RequiredPeerCredential; +import com.yahoo.security.tls.policy.RequiredPeerCredential.Field; +import com.yahoo.security.tls.policy.Role; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Set; + +import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; +import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.CN; +import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.SAN_DNS; +import static java.util.Collections.emptySet; +import static java.util.stream.Collectors.toSet; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + */ +public class PeerAuthorizerTest { + + private static final KeyPair KEY_PAIR = KeyUtils.generateKeypair(KeyAlgorithm.EC); + private static final String ROLE_1 = "role-1", ROLE_2 = "role-2", ROLE_3 = "role-3", POLICY_1 = "policy-1", POLICY_2 = "policy-2"; + + @Test + public void certificate_must_match_both_san_and_cn_pattern() { + RequiredPeerCredential cnRequirement = createRequiredCredential(CN, "*.matching.cn"); + RequiredPeerCredential sanRequirement = createRequiredCredential(SAN_DNS, "*.matching.san"); + PeerAuthorizer authorizer = createPeerAuthorizer(createPolicy(POLICY_1, createRoles(ROLE_1), cnRequirement, sanRequirement)); + + AuthorizationResult result = authorizer.authorizePeer(createCertificate("foo.matching.cn", "foo.matching.san", "foo.invalid.san")); + assertAuthorized(result); + assertThat(result.assumedRoles()).extracting(Role::name).containsOnly(ROLE_1); + assertThat(result.matchedPolicies()).containsOnly(POLICY_1); + + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.invalid.cn", "foo.matching.san"))); + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.invalid.cn", "foo.matching.san", "foo.invalid.san"))); + assertUnauthorized(authorizer.authorizePeer(createCertificate("foo.matching.cn", "foo.invalid.san"))); + } + + @Test + public void can_match_multiple_policies() { + RequiredPeerCredential cnRequirement = createRequiredCredential(CN, "*.matching.cn"); + RequiredPeerCredential sanRequirement = createRequiredCredential(SAN_DNS, "*.matching.san"); + + PeerAuthorizer peerAuthorizer = createPeerAuthorizer( + createPolicy(POLICY_1, createRoles(ROLE_1, ROLE_2), cnRequirement, sanRequirement), + createPolicy(POLICY_2, createRoles(ROLE_2, ROLE_3), cnRequirement, sanRequirement)); + + AuthorizationResult result = peerAuthorizer + .authorizePeer(createCertificate("foo.matching.cn", "foo.matching.san")); + assertAuthorized(result); + assertThat(result.assumedRoles()).extracting(Role::name).containsOnly(ROLE_1, ROLE_2, ROLE_3); + assertThat(result.matchedPolicies()).containsOnly(POLICY_1, POLICY_2); + } + + @Test + public void can_match_subset_of_policies() { + PeerAuthorizer peerAuthorizer = createPeerAuthorizer( + createPolicy(POLICY_1, createRoles(ROLE_1), createRequiredCredential(CN, "*.matching.cn")), + createPolicy(POLICY_2, createRoles(ROLE_1, ROLE_2), createRequiredCredential(SAN_DNS, "*.matching.san"))); + + AuthorizationResult result = peerAuthorizer.authorizePeer(createCertificate("foo.invalid.cn", "foo.matching.san")); + assertAuthorized(result); + assertThat(result.assumedRoles()).extracting(Role::name).containsOnly(ROLE_1, ROLE_2); + assertThat(result.matchedPolicies()).containsOnly(POLICY_2); + } + + @Test + public void must_match_all_cn_and_san_patterns() { + RequiredPeerCredential cnSuffixRequirement = createRequiredCredential(CN, "*.*.matching.suffix.cn"); + RequiredPeerCredential cnPrefixRequirement = createRequiredCredential(CN, "matching.prefix.*.*.*"); + RequiredPeerCredential sanPrefixRequirement = createRequiredCredential(SAN_DNS, "*.*.matching.suffix.san"); + RequiredPeerCredential sanSuffixRequirement = createRequiredCredential(SAN_DNS, "matching.prefix.*.*.*"); + PeerAuthorizer peerAuthorizer = createPeerAuthorizer( + createPolicy(POLICY_1, emptySet(), cnSuffixRequirement, cnPrefixRequirement, sanPrefixRequirement, sanSuffixRequirement)); + + assertAuthorized(peerAuthorizer.authorizePeer(createCertificate("matching.prefix.matching.suffix.cn", "matching.prefix.matching.suffix.san"))); + assertUnauthorized(peerAuthorizer.authorizePeer(createCertificate("matching.prefix.matching.suffix.cn", "matching.prefix.invalid.suffix.san"))); + assertUnauthorized(peerAuthorizer.authorizePeer(createCertificate("invalid.prefix.matching.suffix.cn", "matching.prefix.matching.suffix.san"))); + } + + private static X509Certificate createCertificate(String subjectCn, String... sanCns) { + X509CertificateBuilder builder = + X509CertificateBuilder.fromKeypair( + KEY_PAIR, + new X500Principal("CN=" + subjectCn), + Instant.EPOCH, + Instant.EPOCH.plus(100000, ChronoUnit.DAYS), + SHA256_WITH_ECDSA, + BigInteger.ONE); + for (String sanCn : sanCns) { + builder.addSubjectAlternativeName(sanCn); + } + return builder.build(); + } + + private static RequiredPeerCredential createRequiredCredential(Field field, String pattern) { + return new RequiredPeerCredential(field, new HostGlobPattern(pattern)); + } + + private static Set<Role> createRoles(String... roleNames) { + return Arrays.stream(roleNames).map(Role::new).collect(toSet()); + } + + private static PeerAuthorizer createPeerAuthorizer(PeerPolicy... policies) { + return new PeerAuthorizer(new AuthorizedPeers(Arrays.stream(policies).collect(toSet()))); + } + + private static PeerPolicy createPolicy(String name, Set<Role> roles, RequiredPeerCredential... requiredCredentials) { + return new PeerPolicy(name, roles, Arrays.asList(requiredCredentials)); + } + + private static void assertAuthorized(AuthorizationResult result) { + assertTrue(result.succeeded()); + } + + private static void assertUnauthorized(AuthorizationResult result) { + assertFalse(result.succeeded()); + } + +} diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 3c82931467e..a8771ddc28a 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -61,7 +61,8 @@ class TwoPhaseUpdateOperationTest : public CppUnit::TestFixture, CPPUNIT_TEST(testSafePathConditionMatchSendsPutsWithUpdatedDoc); CPPUNIT_TEST(testSafePathConditionParseFailureFailsWithIllegalParamsError); CPPUNIT_TEST(testSafePathConditonUnknownDocTypeFailsWithIllegalParamsError); - CPPUNIT_TEST(testSafePathConditionWithMissingDocFailsWithTasError); + CPPUNIT_TEST(safe_path_condition_with_missing_doc_and_no_auto_create_fails_with_tas_error); + CPPUNIT_TEST(safe_path_condition_with_missing_doc_and_auto_create_sends_puts); CPPUNIT_TEST(testFastPathCloseEdgeSendsCorrectReply); CPPUNIT_TEST(testSafePathCloseEdgeSendsCorrectReply); CPPUNIT_TEST_SUITE_END(); @@ -97,7 +98,8 @@ protected: void testSafePathConditionMatchSendsPutsWithUpdatedDoc(); void testSafePathConditionParseFailureFailsWithIllegalParamsError(); void testSafePathConditonUnknownDocTypeFailsWithIllegalParamsError(); - void testSafePathConditionWithMissingDocFailsWithTasError(); + void safe_path_condition_with_missing_doc_and_no_auto_create_fails_with_tas_error(); + void safe_path_condition_with_missing_doc_and_auto_create_sends_puts(); void testFastPathCloseEdgeSendsCorrectReply(); void testSafePathCloseEdgeSendsCorrectReply(); @@ -1096,7 +1098,7 @@ TwoPhaseUpdateOperationTest::testSafePathConditonUnknownDocTypeFailsWithIllegalP } void -TwoPhaseUpdateOperationTest::testSafePathConditionWithMissingDocFailsWithTasError() +TwoPhaseUpdateOperationTest::safe_path_condition_with_missing_doc_and_no_auto_create_fails_with_tas_error() { setupDistributor(2, 2, "storage:2 distributor:1"); std::shared_ptr<TwoPhaseUpdateOperation> cb( @@ -1118,6 +1120,22 @@ TwoPhaseUpdateOperationTest::testSafePathConditionWithMissingDocFailsWithTasErro } void +TwoPhaseUpdateOperationTest::safe_path_condition_with_missing_doc_and_auto_create_sends_puts() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions() + .condition("testdoctype1.headerval==120") + .createIfNonExistent(true))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + replyToGet(*cb, sender, 0, 100, false); + replyToGet(*cb, sender, 1, 110, false); + CPPUNIT_ASSERT_EQUAL("Put => 1,Put => 0"s, sender.getCommands(true, false, 2)); +} + +void TwoPhaseUpdateOperationTest::assertAbortedUpdateReplyWithContextPresent( const MessageSenderStub& closeSender) const { @@ -1177,9 +1195,6 @@ TwoPhaseUpdateOperationTest::testSafePathCloseEdgeSendsCorrectReply() // document IDs without explicit doctypes will _not_ be auto-failed on the // distributor. -// XXX shouldn't be necessary to have any special handling of create-if... and -// test-and-set right? They appear fully mutually exclusive. - // XXX: test case where update reply has been sent but callback still // has pending messages (e.g. n-of-m case). diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 686e10ba5ef..197aa95fc22 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -66,8 +66,10 @@ public: void conditional_remove_executed_on_condition_match(); void conditional_update_not_executed_on_condition_mismatch(); void conditional_update_executed_on_condition_match(); + void conditional_update_not_executed_when_no_document_and_no_auto_create(); + void conditional_update_executed_when_no_document_but_auto_create_is_enabled(); void invalid_document_selection_should_fail(); - void non_existing_document_should_fail(); + void conditional_put_to_non_existing_document_should_fail(); void document_with_no_type_should_fail(); CPPUNIT_TEST_SUITE(TestAndSetTest); @@ -77,16 +79,17 @@ public: CPPUNIT_TEST(conditional_remove_executed_on_condition_match); CPPUNIT_TEST(conditional_update_not_executed_on_condition_mismatch); CPPUNIT_TEST(conditional_update_executed_on_condition_match); + CPPUNIT_TEST(conditional_update_not_executed_when_no_document_and_no_auto_create); + CPPUNIT_TEST(conditional_update_executed_when_no_document_but_auto_create_is_enabled); CPPUNIT_TEST(invalid_document_selection_should_fail); - CPPUNIT_TEST(non_existing_document_should_fail); + CPPUNIT_TEST(conditional_put_to_non_existing_document_should_fail); CPPUNIT_TEST(document_with_no_type_should_fail); CPPUNIT_TEST_SUITE_END(); protected: std::unique_ptr<api::UpdateCommand> conditional_update_test( - bool matchingHeader, - api::Timestamp timestampOne, - api::Timestamp timestampTwo); + bool createIfMissing, + api::Timestamp updateTimestamp); document::Document::SP createTestDocument(); document::Document::SP retrieveTestDocument(); @@ -183,18 +186,16 @@ void TestAndSetTest::conditional_remove_executed_on_condition_match() } std::unique_ptr<api::UpdateCommand> TestAndSetTest::conditional_update_test( - bool matchingHeader, - api::Timestamp timestampOne, - api::Timestamp timestampTwo) + bool createIfMissing, + api::Timestamp updateTimestamp) { - putTestDocument(matchingHeader, timestampOne); - auto docUpdate = std::make_shared<document::DocumentUpdate>(_env->_testDocMan.getTypeRepo(), testDoc->getType(), testDocId); auto fieldUpdate = document::FieldUpdate(testDoc->getField("content")); fieldUpdate.addUpdate(document::AssignValueUpdate(NEW_CONTENT)); docUpdate->addUpdate(fieldUpdate); + docUpdate->setCreateIfNonExistent(createIfMissing); - auto updateUp = std::make_unique<api::UpdateCommand>(makeDocumentBucket(BUCKET_ID), docUpdate, timestampTwo); + auto updateUp = std::make_unique<api::UpdateCommand>(makeDocumentBucket(BUCKET_ID), docUpdate, updateTimestamp); setTestCondition(*updateUp); return updateUp; } @@ -203,11 +204,12 @@ void TestAndSetTest::conditional_update_not_executed_on_condition_mismatch() { api::Timestamp timestampOne = 0; api::Timestamp timestampTwo = 1; - auto updateUp = conditional_update_test(false, timestampOne, timestampTwo); + putTestDocument(false, timestampOne); + auto updateUp = conditional_update_test(false, timestampTwo); CPPUNIT_ASSERT(thread->handleUpdate(*updateUp)->getResult() == api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), - dumpBucket(BUCKET_ID)); + dumpBucket(BUCKET_ID)); assertTestDocumentFoundAndMatchesContent(OLD_CONTENT); } @@ -216,7 +218,8 @@ void TestAndSetTest::conditional_update_executed_on_condition_match() { api::Timestamp timestampOne = 0; api::Timestamp timestampTwo = 1; - auto updateUp = conditional_update_test(true, timestampOne, timestampTwo); + putTestDocument(true, timestampOne); + auto updateUp = conditional_update_test(false, timestampTwo); CPPUNIT_ASSERT(thread->handleUpdate(*updateUp)->getResult() == api::ReturnCode::Result::OK); CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId) + @@ -226,6 +229,23 @@ void TestAndSetTest::conditional_update_executed_on_condition_match() assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); } +void TestAndSetTest::conditional_update_not_executed_when_no_document_and_no_auto_create() { + api::Timestamp updateTimestamp = 200; + auto updateUp = conditional_update_test(false, updateTimestamp); + + CPPUNIT_ASSERT(thread->handleUpdate(*updateUp)->getResult() == api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); + CPPUNIT_ASSERT_EQUAL(""s, dumpBucket(BUCKET_ID)); +} + +void TestAndSetTest::conditional_update_executed_when_no_document_but_auto_create_is_enabled() { + api::Timestamp updateTimestamp = 200; + auto updateUp = conditional_update_test(true, updateTimestamp); + + CPPUNIT_ASSERT(thread->handleUpdate(*updateUp)->getResult() == api::ReturnCode::Result::OK); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(updateTimestamp, testDocId), dumpBucket(BUCKET_ID)); + assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); +} + void TestAndSetTest::invalid_document_selection_should_fail() { // Conditionally replace nonexisting document @@ -238,7 +258,7 @@ void TestAndSetTest::invalid_document_selection_should_fail() CPPUNIT_ASSERT_EQUAL(""s, dumpBucket(BUCKET_ID)); } -void TestAndSetTest::non_existing_document_should_fail() +void TestAndSetTest::conditional_put_to_non_existing_document_should_fail() { // Conditionally replace nonexisting document // Fail since no document exists to match with test and set diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index c652f787b2e..b7ebafc114c 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -378,7 +378,7 @@ TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorMessageSender& sen } docToUpdate = reply.getDocument(); setUpdatedForTimestamp(receivedTimestamp); - } else if (hasTasCondition()) { + } else if (hasTasCondition() && !shouldCreateIfNonExistent()) { replyWithTasFailure(sender, "Document did not exist"); return; } else if (shouldCreateIfNonExistent()) { diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 1b221ea0b7c..55f7c076244 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -85,9 +85,10 @@ bool PersistenceThread::tasConditionExists(const api::TestAndSetCommand & cmd) { return cmd.getCondition().isPresent(); } -bool PersistenceThread::tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker) { +bool PersistenceThread::tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, + bool missingDocumentImpliesMatch) { try { - TestAndSetHelper helper(*this, cmd); + TestAndSetHelper helper(*this, cmd, missingDocumentImpliesMatch); auto code = helper.retrieveAndMatch(); if (code.failed()) { @@ -149,7 +150,7 @@ PersistenceThread::handleUpdate(api::UpdateCommand& cmd) auto tracker = std::make_unique<MessageTracker>(metrics, _env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker)) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, cmd.getUpdate()->getCreateIfNonExistent())) { return tracker; } diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index 05ca34f623d..ed27a759e8b 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -87,7 +87,8 @@ private: friend class TestAndSetHelper; bool tasConditionExists(const api::TestAndSetCommand & cmd); - bool tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker); + bool tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, + bool missingDocumentImpliesMatch = false); }; } // storage diff --git a/storage/src/vespa/storage/persistence/testandsethelper.cpp b/storage/src/vespa/storage/persistence/testandsethelper.cpp index 70a6bdf21db..511d44ad331 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.cpp +++ b/storage/src/vespa/storage/persistence/testandsethelper.cpp @@ -39,11 +39,13 @@ spi::GetResult TestAndSetHelper::retrieveDocument(const document::FieldSet & fie _thread._context); } -TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd) +TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd, + bool missingDocumentImpliesMatch) : _thread(thread), _component(thread._env._component), _cmd(cmd), - _docId(cmd.getDocumentId()) + _docId(cmd.getDocumentId()), + _missingDocumentImpliesMatch(missingDocumentImpliesMatch) { getDocumentType(); parseDocumentSelection(); @@ -69,6 +71,8 @@ api::ReturnCode TestAndSetHelper::retrieveAndMatch() { // Document matches return api::ReturnCode(); + } else if (_missingDocumentImpliesMatch) { + return api::ReturnCode(); } return api::ReturnCode(api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, "Document does not exist"); diff --git a/storage/src/vespa/storage/persistence/testandsethelper.h b/storage/src/vespa/storage/persistence/testandsethelper.h index 4e48259bc79..21c111c712f 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.h +++ b/storage/src/vespa/storage/persistence/testandsethelper.h @@ -26,13 +26,15 @@ class TestAndSetHelper { const document::DocumentId _docId; const document::DocumentType * _docTypePtr; std::unique_ptr<document::select::Node> _docSelectionUp; + bool _missingDocumentImpliesMatch; void getDocumentType(); void parseDocumentSelection(); spi::GetResult retrieveDocument(const document::FieldSet & fieldSet); public: - TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd); + TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd, + bool missingDocumentImpliesMatch = false); ~TestAndSetHelper(); api::ReturnCode retrieveAndMatch(); }; |