diff options
138 files changed, 1239 insertions, 1147 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 2de11be08f2..db01bb91b3b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -24,7 +24,7 @@ public enum SystemName { Public, /** VaaS */ - vaas; + vaas; // TODO: Remove this and use public everywhere public static SystemName defaultSystem() { return main; @@ -35,7 +35,7 @@ public enum SystemName { case "dev": return dev; case "cd": return cd; case "main": return main; - case "public": return Public; + case "public": case "Public": return Public; case "vaas": return vaas; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java index aba3b5f3ab7..fffa849f7d3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; @@ -10,7 +11,7 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; */ public interface AthenzClientFactory { - AthenzService getControllerIdentity(); + AthenzIdentity getControllerIdentity(); ZmsClient createZmsClient(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index c168ddf6caf..db9291cd651 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -53,7 +54,7 @@ public interface ZoneRegistry { SystemName system(); /** Return the configserver's Athenz service identity */ - AthenzService getConfigServerAthenzService(ZoneId zoneId); + AthenzIdentity getConfigServerAthenzIdentity(ZoneId zoneId); /** Returns the Vespa upgrade policy to use for zones in this registry */ UpgradePolicy upgradePolicy(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java index 3ba0367a00c..14d8d06d0c6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java @@ -9,7 +9,7 @@ import java.util.Objects; import java.util.Optional; /** - * The context in which a role is valid. + * The context in which a role is valid. This is immutable. * * @author mpolden */ @@ -40,11 +40,6 @@ public class Context { return system; } - /** Returns whether this context is considered limited */ - public boolean limited() { - return tenant.isPresent() || application.isPresent(); - } - /** Returns a context that has no restrictions on tenant or application in given system */ public static Context unlimitedIn(SystemName system) { return new Context(Optional.empty(), Optional.empty(), system); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index ff535e92033..c28fa7a3fc3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.api.role; import java.net.URI; import java.util.Objects; -import static java.util.Objects.requireNonNull; - /** * A role is a combination of a {@link RoleDefinition} and a {@link Context}, which allows evaluation * of access control for a given action on a resource. Create using {@link Roles}. @@ -18,15 +16,15 @@ public abstract class Role { final Context context; Role(RoleDefinition roleDefinition, Context context) { - this.roleDefinition = requireNonNull(roleDefinition); - this.context = requireNonNull(context); + this.roleDefinition = Objects.requireNonNull(roleDefinition); + this.context = Objects.requireNonNull(context); } /** Returns the role definition of this bound role. */ public RoleDefinition definition() { return roleDefinition; } /** Returns whether this role is allowed to perform the given action on the given resource. */ - public boolean allows(Action action, URI uri) { + public final boolean allows(Action action, URI uri) { return roleDefinition.policies().stream().anyMatch(policy -> policy.evaluate(action, uri, context)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java index 41444258a68..3378f9e0061 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java @@ -5,8 +5,9 @@ import java.security.Principal; import java.util.Objects; import java.util.Set; -import static java.util.Objects.requireNonNull; - +/** + * @author tokle + */ public class SecurityContext { public static final String ATTRIBUTE_NAME = SecurityContext.class.getName(); @@ -15,7 +16,7 @@ public class SecurityContext { private final Set<Role> roles; public SecurityContext(Principal principal, Set<Role> roles) { - this.principal = requireNonNull(principal); + this.principal = Objects.requireNonNull(principal); this.roles = Set.copyOf(roles); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 3664f3712e1..84e15deea4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -9,16 +9,16 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -199,9 +199,10 @@ public class Application { return rotation; } - /** Returns the global rotation dns name, if present */ - public Optional<GlobalDnsName> globalDnsName(SystemName system) { - return rotation.map(ignored -> new GlobalDnsName(id, system)); + /** Returns the default global endpoints for this in given system */ + public EndpointList endpointsIn(SystemName system) { + if (rotation.isEmpty()) return EndpointList.EMPTY; + return EndpointList.defaultGlobal(id, system); } public Optional<String> pemDeployKey() { return pemDeployKey; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7caefa55d28..4c5be570a02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -45,7 +45,8 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; @@ -283,7 +284,7 @@ public class ApplicationController { ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set<String> rotationNames = new HashSet<>(); - Set<String> cnames = new HashSet<>(); + Set<String> cnames; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -324,13 +325,7 @@ public class ApplicationController { // Assign global rotation application = withRotation(application, zone); Application app = application.get(); - app.globalDnsName(controller.system()).ifPresent(applicationRotation -> { - rotationNames.add(app.rotation().orElseThrow(() -> new RuntimeException("Global Dns assigned, but no rotation id present")).asString()); - cnames.add(applicationRotation.dnsName()); - cnames.add(applicationRotation.secureDnsName()); - cnames.add(applicationRotation.oathDnsName()); - }); - + cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); // Update application with information from application package if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally() @@ -438,18 +433,20 @@ public class ApplicationController { application = application.with(rotation.id()); store(application); // store assigned rotation even if deployment fails - GlobalDnsName dnsName = application.get().globalDnsName(controller.system()) - .orElseThrow(() -> new IllegalStateException("Expected rotation to be assigned")); boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) .value(); - registerCname(dnsName.oathDnsName(), rotation.name()); - if (redirectLegacyDns) { - registerCname(dnsName.dnsName(), dnsName.oathDnsName()); - registerCname(dnsName.secureDnsName(), dnsName.oathDnsName()); - } else { - registerCname(dnsName.dnsName(), rotation.name()); - registerCname(dnsName.secureDnsName(), rotation.name()); - } + + EndpointList globalEndpoints = application.get() + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + globalEndpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + if (redirectLegacyDns) { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); + } else { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); + } + }); } } return application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java new file mode 100644 index 00000000000..14a5d3c7ddf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -0,0 +1,281 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.net.URI; +import java.util.Objects; + +/** + * Represents an application's endpoint. The endpoint scope can either be global or a specific zone. This is visible to + * the tenant and is used by the tenant when accessing deployments. + * + * @author mpolden + */ +public class Endpoint { + + public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; + public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; + + private final URI url; + private final Scope scope; + private final boolean legacy; + private final boolean directRouting; + + private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy, + boolean directRouting) { + Objects.requireNonNull(name, "name must be non-null"); + Objects.requireNonNull(application, "application must be non-null"); + Objects.requireNonNull(system, "system must be non-null"); + Objects.requireNonNull(port, "port must be non-null"); + this.url = createUrl(name, application, zone, system, port, legacy, directRouting); + this.scope = zone == null ? Scope.global : Scope.zone; + this.legacy = legacy; + this.directRouting = directRouting; + } + + /** Returns the URL used to access this */ + public URI url() { + return url; + } + + /** Returns the DNS name of this */ + public String dnsName() { + return url.getHost(); + } + + /** Returns the scope of this */ + public Scope scope() { + return scope; + } + + /** Returns whether this is considered a legacy DNS name that is due for removal */ + public boolean legacy() { + return legacy; + } + + /** + * Returns whether this endpoint supports direct routing. Direct routing means that this endpoint is served by an + * exclusive load balancer instead of a shared routing layer. + */ + public boolean directRouting() { + return directRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Endpoint endpoint = (Endpoint) o; + return url.equals(endpoint.url); + } + + @Override + public int hashCode() { + return Objects.hash(url); + } + + @Override + public String toString() { + return String.format("endpoint %s [scope=%s, legacy=%s, directRouting=%s]", url, scope, legacy, directRouting); + } + + private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system, + Port port, boolean legacy, boolean directRouting) { + String scheme = port.tls ? "https" : "http"; + String separator = separator(system, directRouting, port.tls); + String portPart = port.isDefault() ? "" : ":" + port.port; + return URI.create(scheme + "://" + + sanitize(namePart(name, separator)) + + systemPart(system, separator) + + sanitize(instancePart(application, zone, separator)) + + sanitize(application.application().value()) + + separator + + sanitize(application.tenant().value()) + + "." + + scopePart(zone, legacy) + + dnsSuffix(system, legacy) + + portPart + + "/"); + } + + private static String sanitize(String part) { // TODO: Reject reserved words + return part.replace('_', '-'); + } + + private static String separator(SystemName system, boolean directRouting, boolean tls) { + if (!tls) return "."; + if (directRouting) return "."; + if (isPublic(system)) return "."; + return "--"; + } + + private static String namePart(String name, String separator) { + if ("default".equals(name)) return ""; + return name + separator; + } + + private static String scopePart(ZoneId zone, boolean legacy) { + if (zone == null) return "global"; + if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints + return zone.region().value() + "." + zone.environment().value(); + } + + private static String instancePart(ApplicationId application, ZoneId zone, String separator) { + if (zone == null) return ""; // Always omit instance for global endpoints + if (application.instance().isDefault()) return ""; // Skip "default" + return application.instance().value() + separator; + } + + private static String systemPart(SystemName system, String separator) { + if (system == SystemName.main || isPublic(system)) return ""; + return system.name() + separator; + } + + private static String dnsSuffix(SystemName system, boolean legacy) { + switch (system) { + case cd: + case main: + if (legacy) return YAHOO_DNS_SUFFIX; + return OATH_DNS_SUFFIX; + case Public: + case vaas: + return PUBLIC_DNS_SUFFIX; + default: throw new IllegalArgumentException("No DNS suffix declared for system " + system); + } + } + + private static boolean isPublic(SystemName system) { // TODO: Remove and inline once we're down to one + return system == SystemName.Public || system == SystemName.vaas; + } + + /** An endpoint's scope */ + public enum Scope { + + /** Endpoint points to all zones */ + global, + + /** Endpoint points to a single zone */ + zone, + + } + + /** Represents an endpoint's HTTP port */ + public static class Port { + + private final int port; + private final boolean tls; + + private Port(int port, boolean tls) { + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); + } + this.port = port; + this.tls = tls; + } + + private boolean isDefault() { + return port == 80 || port == 443; + } + + /** Returns the default HTTPS port */ + public static Port tls() { + return new Port(443, true); + } + + /** Create a HTTPS port */ + public static Port tls(int port) { + return new Port(port, true); + } + + /** Create a HTTP port */ + public static Port plain(int port) { + return new Port(port, false); + } + + } + + /** Build an endpoint for given application */ + public static EndpointBuilder of(ApplicationId application) { + return new EndpointBuilder(application); + } + + public static class EndpointBuilder { + + private final ApplicationId application; + + private ZoneId zone; + private ClusterSpec.Id cluster; + private RotationName rotation; + private Port port; + private boolean legacy = false; + private boolean directRouting = false; + + private EndpointBuilder(ApplicationId application) { + this.application = application; + } + + /** Sets the cluster and zone target of this */ + public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { + if (rotation != null) { + throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + } + this.cluster = cluster; + this.zone = zone; + return this; + } + + /** Sets the rotation target of this */ + public EndpointBuilder target(RotationName rotation) { + if (cluster != null && zone != null) { + throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + } + this.rotation = rotation; + return this; + } + + /** Sets the port of this */ + public EndpointBuilder on(Port port) { + this.port = port; + return this; + } + + /** Marks this as a legacy endpoint */ + public EndpointBuilder legacy() { + this.legacy = true; + return this; + } + + /** Enables direct routing support for this */ + public EndpointBuilder directRouting() { + this.directRouting = true; + return this; + } + + /** Sets the system that owns this */ + public Endpoint in(SystemName system) { + String name; + if (cluster != null && zone != null) { + name = cluster.value(); + } else if (rotation != null) { + name = rotation.value(); + } else { + throw new IllegalArgumentException("Must set either cluster or rotation target"); + } + if (isPublic(system) && !directRouting) { + throw new IllegalArgumentException("Public system only supports direct routing endpoints"); + } + if (directRouting && !port.isDefault()) { + throw new IllegalArgumentException("Direct routing endpoints only support default port"); + } + return new Endpoint(name, application, zone, system, port, legacy, directRouting); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java new file mode 100644 index 00000000000..0c04a1f099c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -0,0 +1,85 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * A list of endpoints for an application. + * + * @author mpolden + */ +public class EndpointList { + + public static final EndpointList EMPTY = new EndpointList(List.of()); + + private final List<Endpoint> endpoints; + + private EndpointList(List<Endpoint> endpoints) { + long mainEndpoints = endpoints.stream() + .filter(endpoint -> endpoint.scope() == Endpoint.Scope.global) + .filter(Predicate.not(Endpoint::directRouting)) + .filter(Predicate.not(Endpoint::legacy)).count(); + if (mainEndpoints > 1) { + throw new IllegalArgumentException("Can have only 1 non-legacy global endpoint, got " + endpoints); + } + if (endpoints.stream().distinct().count() != endpoints.size()) { + throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); + } + this.endpoints = List.copyOf(endpoints); + } + + public List<Endpoint> asList() { + return endpoints; + } + + /** Returns the main endpoint, if any */ + public Optional<Endpoint> main() { + return endpoints.stream().filter(Predicate.not(Endpoint::legacy)).findFirst(); + } + + /** Returns the subset of endpoints are either legacy or not */ + public EndpointList legacy(boolean legacy) { + return of(endpoints.stream().filter(endpoint -> endpoint.legacy() == legacy)); + } + + /** Returns the subset of endpoints with given scope */ + public EndpointList scope(Endpoint.Scope scope) { + return of(endpoints.stream().filter(endpoint -> endpoint.scope() == scope)); + } + + /** Returns the union of this and given endpoints */ + public EndpointList and(EndpointList endpoints) { + return of(Stream.concat(asList().stream(), endpoints.asList().stream())); + } + + public static EndpointList of(Stream<Endpoint> endpoints) { + return new EndpointList(endpoints.collect(Collectors.toUnmodifiableList())); + } + + /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ + public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { + // Rotation name is always default in the routing layer + RotationName rotation = RotationName.from("default"); + switch (system) { + case cd: + case main: + return new EndpointList(List.of( + Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + )); + } + return EMPTY; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java deleted file mode 100644 index ae638beed5c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.RotationName; -import com.yahoo.config.provision.SystemName; - -import java.net.URI; -import java.util.Objects; - -/** - * Represents names for an application's global rotation. - * - * @author mpolden - */ -public class GlobalDnsName { - - // TODO: TLS: Remove all non-secure stuff when all traffic is on HTTPS. - public static final String DNS_SUFFIX = "global.vespa.yahooapis.com"; - public static final String OATH_DNS_SUFFIX = "global.vespa.oath.cloud"; - private static final int port = 4080; - private static final int securePort = 4443; - - private final URI url; - private final URI secureUrl; - private final URI oathUrl; - - public GlobalDnsName(ApplicationId application, SystemName system) { - this(application, system, null); - } - - public GlobalDnsName(ApplicationId application, SystemName system, RotationName rotation) { - Objects.requireNonNull(application, "application must be non-null"); - Objects.requireNonNull(system, "system must be non-null"); - - this.url = URI.create(String.format("http://%s%s%s.%s.%s:%d/", - clusterPart(rotation, "."), - systemPart(system, "."), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - DNS_SUFFIX, - port)); - this.secureUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/", - clusterPart(rotation, "--"), - systemPart(system, "--"), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - DNS_SUFFIX, - securePort)); - this.oathUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/", - clusterPart(rotation, "--"), - systemPart(system, "--"), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - OATH_DNS_SUFFIX, - securePort)); - } - - /** URL to this rotation */ - public URI url() { - return url; - } - - /** HTTPS URL to this rotation */ - public URI secureUrl() { - return secureUrl; - } - - /** Oath HTTPS URL to this rotation */ - public URI oathUrl() { - return oathUrl; - } - - /** DNS name for this rotation */ - public String dnsName() { - return url.getHost(); - } - - /** DNS name for this rotation */ - public String secureDnsName() { - return secureUrl.getHost(); - } - - /** Oath DNS name for this rotation */ - public String oathDnsName() { - return oathUrl.getHost(); - } - - /** Sanitize by translating '_' to '-' as the former is not allowed in a DNS name */ - private static String sanitize(String s) { - return s.replace('_', '-'); - } - - private static String clusterPart(RotationName rotation, String separator) { - return rotation == null ? "" : rotation.value() + separator; - } - - private static String systemPart(SystemName system, String separator) { - return SystemName.main == system ? "" : system.name() + separator; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java index c459d519ab6..c4b69ce5588 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java @@ -6,14 +6,13 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Represents the DNS routing policy for a load balancer. @@ -41,9 +40,9 @@ public class RoutingPolicy { this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); } - public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, HostName canonicalName, + public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, SystemName system, HostName canonicalName, Optional<String> dnsZone, Set<RotationName> rotations) { - this(owner, zone, HostName.from(aliasOf(cluster, owner, zone)), canonicalName, dnsZone, rotations); + this(owner, zone, HostName.from(endpointOf(cluster, owner, zone, system).dnsName()), canonicalName, dnsZone, rotations); } /** The application owning this */ @@ -76,6 +75,11 @@ public class RoutingPolicy { return rotations; } + /** Endpoints for this routing policy */ + public EndpointList endpointsIn(SystemName system) { + return EndpointList.of(rotations.stream().map(rotation -> endpointOf(owner, rotation, system))); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -98,21 +102,14 @@ public class RoutingPolicy { zone.value()); } - /** Returns the alias to use for the given application cluster in zone */ - private static String aliasOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone) { - List<String> parts = List.of(ignorePartIfDefault(cluster.value()), - ignorePartIfDefault(application.instance().value()), - application.application().value(), - application.tenant().value() + - "." + zone.value() + "." + "vespa.oath.cloud" - ); - return parts.stream() - .filter(Predicate.not(String::isBlank)) - .collect(Collectors.joining("--")); + /** Returns the endpoint of given rotation */ + public static Endpoint endpointOf(ApplicationId application, RotationName rotation, SystemName system) { + return Endpoint.of(application).target(rotation).on(Port.tls()).directRouting().in(system); } - private static String ignorePartIfDefault(String s) { - return "default".equalsIgnoreCase(s) ? "" : s; + /** Returns the endpoint of given cluster */ + public static Endpoint endpointOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone, SystemName system) { + return Endpoint.of(application).target(cluster, zone).on(Port.tls()).directRouting().in(system); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 846c90a96f5..447f9a462b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -28,7 +29,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { } @Override - public AthenzService getControllerIdentity() { + public AthenzIdentity getControllerIdentity() { return identityProvider.identity(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 0732eeb97c3..75b7e137998 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -19,9 +18,9 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; -import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -45,14 +44,14 @@ public class AthenzFacade implements AccessControl { private static final Logger log = Logger.getLogger(AthenzFacade.class.getName()); private final ZmsClient zmsClient; private final ZtsClient ztsClient; - private final AthenzService service; + private final AthenzIdentity service; @Inject public AthenzFacade(AthenzClientFactory factory) { this(factory.createZmsClient(), factory.createZtsClient(), factory.getControllerIdentity()); } - public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzService identity) { + public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzIdentity identity) { this.zmsClient = zmsClient; this.ztsClient = ztsClient; this.service = identity; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index f7a8e702b06..37926d944b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -30,23 +29,23 @@ public class ZmsClientMock implements ZmsClient { private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName()); private final AthenzDbMock athenz; - private final AthenzService controllerIdentity; + private final AthenzIdentity controllerIdentity; private static final Pattern TENANT_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.(?<tenantDomain>[\\w\\-_]+)\\..*"); private static final Pattern APPLICATION_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.[\\w\\-_]+\\.res_group\\.(?<resourceGroup>[\\w\\-_]+)\\.wildcard"); - public ZmsClientMock(AthenzDbMock athenz, AthenzService controllerIdentity) { + public ZmsClientMock(AthenzDbMock athenz, AthenzIdentity controllerIdentity) { this.athenz = athenz; this.controllerIdentity = controllerIdentity; } @Override - public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { log("createTenancy(tenantDomain='%s')", tenantDomain); getDomainOrThrow(tenantDomain, false).isVespaTenant = true; } @Override - public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { log("deleteTenancy(tenantDomain='%s')", tenantDomain); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, false); domain.isVespaTenant = false; @@ -55,7 +54,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { log("createProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); ApplicationId applicationId = new ApplicationId(resourceGroup); @@ -65,7 +64,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token) { log("deleteProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); getDomainOrThrow(tenantDomain, true).applications.remove(new ApplicationId(resourceGroup)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java index 8bb5ad12468..5c0407d35a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -45,22 +45,22 @@ public class ZtsClientMock implements ZtsClient { } @Override - public InstanceIdentity registerInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { + public InstanceIdentity registerInstance(AthenzIdentity providerIdentity, AthenzIdentity instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public InstanceIdentity refreshInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { + public InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix) { throw new UnsupportedOperationException(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 787a050e59e..0cf89d798a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -65,15 +65,17 @@ public class DeploymentMetricsMaintainer extends Maintainer { .getDeploymentMetrics(application.id(), deployment.zone()); Instant now = controller().clock().instant(); applications.lockIfPresent(application.id(), locked -> { - DeploymentMetrics newMetrics = locked.get().deployments().get(deployment.zone()).metrics() - .withQueriesPerSecond(collectedMetrics.queriesPerSecond()) - .withWritesPerSecond(collectedMetrics.writesPerSecond()) - .withDocumentCount(collectedMetrics.documentCount()) - .withQueryLatencyMillis(collectedMetrics.queryLatencyMillis()) - .withWriteLatencyMillis(collectedMetrics.writeLatencyMillis()) - .at(now); - applications.store(locked.with(deployment.zone(), newMetrics) - .recordActivityAt(now, deployment.zone())); + Deployment existingDeployment = locked.get().deployments().get(deployment.zone()); + if (existingDeployment == null) return; // Deployment removed since we started collecting metrics + DeploymentMetrics newMetrics = existingDeployment.metrics() + .withQueriesPerSecond(collectedMetrics.queriesPerSecond()) + .withWritesPerSecond(collectedMetrics.writesPerSecond()) + .withDocumentCount(collectedMetrics.documentCount()) + .withQueryLatencyMillis(collectedMetrics.queryLatencyMillis()) + .withWriteLatencyMillis(collectedMetrics.writeLatencyMillis()) + .at(now); + applications.store(locked.with(existingDeployment.zone(), newMetrics) + .recordActivityAt(now, existingDeployment.zone())); }); } } catch (Exception e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java index 7693f224b56..c2c68591dea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -84,8 +84,8 @@ public class DnsMaintainer extends Maintainer { /** Returns whether we can update the given record */ private static boolean canUpdate(Record record) { String recordName = record.name().asString(); - return recordName.endsWith(GlobalDnsName.DNS_SUFFIX) || - recordName.endsWith(GlobalDnsName.OATH_DNS_SUFFIX); + return recordName.endsWith(Endpoint.YAHOO_DNS_SUFFIX) || + recordName.endsWith(Endpoint.OATH_DNS_SUFFIX); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java index 9e10c5f9194..417a1944ad3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.RoutingId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -91,7 +91,8 @@ public class RoutingPolicyMaintainer extends Maintainer { // Create DNS record for each routing ID for (Map.Entry<RoutingId, List<RoutingPolicy>> route : routingTable.entrySet()) { - GlobalDnsName dnsName = dnsName(route.getKey()); + Endpoint endpoint = RoutingPolicy.endpointOf(route.getKey().application(), route.getKey().rotation(), + controller().system()); Set<AliasTarget> targets = route.getValue() .stream() .filter(policy -> policy.dnsZone().isPresent()) @@ -100,10 +101,10 @@ public class RoutingPolicyMaintainer extends Maintainer { policy.zone())) .collect(Collectors.toSet()); try { - nameService.createAlias(RecordName.from(dnsName.oathDnsName()), targets); + nameService.createAlias(RecordName.from(endpoint.dnsName()), targets); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to create or update DNS record for global rotation " + - dnsName.oathDnsName() + ". Retrying in " + maintenanceInterval(), e); + endpoint.dnsName() + ". Retrying in " + maintenanceInterval(), e); } } } @@ -136,7 +137,8 @@ public class RoutingPolicyMaintainer extends Maintainer { /** Register DNS alias for given load balancer */ private RoutingPolicy registerCname(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, loadBalancer.cluster(), + RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, + loadBalancer.cluster(), controller().system(), loadBalancer.hostname(), loadBalancer.dnsZone(), loadBalancer.rotations()); RecordName name = RecordName.from(routingPolicy.alias().value()); @@ -186,23 +188,18 @@ public class RoutingPolicyMaintainer extends Maintainer { Set<RoutingId> activeRoutingIds = routingIdsFrom(loadBalancers); removalCandidates.removeAll(activeRoutingIds); for (RoutingId id : removalCandidates) { - GlobalDnsName dnsName = dnsName(id); + Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller().system()); try { - List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(dnsName.oathDnsName())); + List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName())); nameService.removeRecords(records); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + dnsName.oathDnsName() + + log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + endpoint.dnsName() + "'. Retrying in " + maintenanceInterval()); } } } } - /** Create a global DNS name for given routing ID */ - private GlobalDnsName dnsName(RoutingId routingId) { - return new GlobalDnsName(routingId.application(), controller().system(), routingId.rotation()); - } - /** Compute routing IDs from given load balancers */ private static Set<RoutingId> routingIdsFrom(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { Set<RoutingId> routingIds = new LinkedHashSet<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index c223d051237..01d9a01a316 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -268,7 +268,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { AthenzIdentityVerifier hostnameVerifier = new AthenzIdentityVerifier( singleton( - zoneRegistry.getConfigServerAthenzService( + zoneRegistry.getConfigServerAthenzIdentity( ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); return HttpClientBuilder.create() .setUserAgent("config-server-proxy-client") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index b48c8723d44..612c323fc31 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -59,7 +58,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; @@ -482,21 +481,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Rotation Cursor globalRotationsArray = object.setArray("globalRotations"); + application.endpointsIn(controller.system()) + .scope(Endpoint.Scope.global) + .legacy(false) // Hide legacy names + .asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalRotationsArray::addString); - application.globalDnsName(controller.system()).ifPresent(rotation -> { - globalRotationsArray.addString(rotation.url().toString()); - globalRotationsArray.addString(rotation.secureUrl().toString()); - globalRotationsArray.addString(rotation.oathUrl().toString()); - object.setString("rotationId", application.rotation().get().asString()); - }); + application.rotation().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies(application.id()); for (RoutingPolicy policy : routingPolicies) { - for (RotationName rotation : policy.rotations()) { - GlobalDnsName dnsName = new GlobalDnsName(application.id(), controller.system(), rotation); - globalRotationsArray.addString(dnsName.oathUrl().toString()); - } + policy.endpointsIn(controller.system()).asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalRotationsArray::addString); } // Deployments sorted according to deployment spec @@ -574,6 +575,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); + // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may + // contain the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to + // ask the routing layer here Cursor serviceUrlArray = response.setArray("serviceUrls"); controller.applications().getDeploymentEndpoints(deploymentId) .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java new file mode 100644 index 00000000000..16b875c1892 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -0,0 +1,114 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class EndpointTest { + + private static final ApplicationId app1 = ApplicationId.from("t1", "a1", "default"); + private static final ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); + + @Test + public void test_global_endpoints() { + RotationName rotation = RotationName.from("default"); // Always default for non-direct routing + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint + "http://a1.t1.global.vespa.yahooapis.com:4080/", + Endpoint.of(app1).target(rotation).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Legacy endpoint with TLS + "https://a1--t1.global.vespa.yahooapis.com:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Main endpoint + "https://a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).in(SystemName.main), + + // Main endpoint in CD + "https://cd--a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).in(SystemName.cd), + + // Main endpoint with direct routing and default TLS port + "https://a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).target(rotation).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint with custom rotation name + "https://r1.a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).target(RotationName.from("r1")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance in default rotation + "https://a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).target(rotation).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance with custom rotation name + "https://r2.a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).target(RotationName.from("r2")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint in public system + "https://a1.t1.global.public.vespa.oath.cloud/", + Endpoint.of(app1).target(rotation).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + + @Test + public void test_zone_endpoints() { + ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing + ZoneId prodZone = ZoneId.from("prod", "us-north-1"); + ZoneId testZone = ZoneId.from("test", "us-north-2"); + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint (always contains environment) + "http://a1.t1.us-north-1.prod.vespa.yahooapis.com:4080/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Secure legacy endpoint + "https://a1--t1.us-north-1.prod.vespa.yahooapis.com:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Prod endpoint in main + "https://a1--t1.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + + // Prod endpoint in CD + "https://cd--a1--t1.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.cd), + + // Test endpoint in main + "https://a1--t1.us-north-2.test.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, testZone).on(Port.tls(4443)).in(SystemName.main), + + // Non-default cluster in main + "https://c1--a1--t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).in(SystemName.main), + + // Non-default instance in main + "https://i2--a2--t2.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app2).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + + // Non-default cluster in public + "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/", + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public), + + // Non-default cluster and instance in public + "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/", + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java deleted file mode 100644 index 56c2f9bd968..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import org.junit.Test; - -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class RoutingPolicyTest { - - @Test - public void test_endpoint_names() { - ZoneId zoneId = ZoneId.from("prod", "us-north-1"); - ApplicationId withInstance = ApplicationId.from("tenant", "application", "instance"); - testAlias("instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withInstance, zoneId); - testAlias("cluster--instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withInstance, zoneId); - - ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); - testAlias("application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); - testAlias("cluster--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); - } - - private void testAlias(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { - assertEquals(expected, new RoutingPolicy(applicationId, zoneId, ClusterSpec.Id.from(clusterName), - HostName.from("lb-0"), Optional.empty(), Set.of()).alias().value()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 393268b4750..4248a513950 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -101,7 +101,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return ZoneFilterMock.from(Collections.unmodifiableList(zones)); } - public AthenzService getConfigServerAthenzService(ZoneId zone) { + public AthenzService getConfigServerAthenzIdentity(ZoneId zone) { return new AthenzService("vespadomain", "provider-" + zone.environment().value() + "-" + zone.region().value()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index cbf50b65d1a..256ace4ae09 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -117,7 +117,7 @@ public class DnsMaintainerTest { for (int i = 1; i <= staleTotal; i++) { Rotation r = rotation(i); tester.controllerTester().nameService().createCname(RecordName.from("stale-record-" + i + "." + - GlobalDnsName.OATH_DNS_SUFFIX), + Endpoint.OATH_DNS_SUFFIX), RecordData.from(r.name() + ".")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java index 47d507e6094..0541a0b05f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java @@ -6,11 +6,12 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -56,15 +57,22 @@ public class RoutingPolicyMaintainerTest { public void maintains_global_routing_policies() { int clustersPerZone = 2; tester.deployCompletely(app1, applicationPackage); - Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0"))); + // Cluster is member of 2 global rotations + Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0"), RotationName.from("r1"))); provisionLoadBalancers(app1, clustersPerZone, rotations); - // Creates alias record for cluster0 + // Creates alias records for cluster0 maintainer.maintain(); - Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app1--tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app1.tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r1.app1.tenant1.global.vespa.oath.cloud")); assertEquals(2, records1.get().size()); + assertEquals(records1.get().size(), records2.get().size()); assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records1.get().get(0).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records2.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records2.get().get(1).data().asString()); + assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() + .endpointsIn(SystemName.main).asList().size()); // Applications gains a new deployment ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() @@ -85,13 +93,13 @@ public class RoutingPolicyMaintainerTest { assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(2).data().asString()); // Another application is deployed - Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app2--tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records3 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app2.tenant1.global.vespa.oath.cloud")); tester.deployCompletely(app2, applicationPackage); provisionLoadBalancers(app2, 1, Map.of(0, Set.of(RotationName.from("r0")))); maintainer.maintain(); - assertEquals(2, records2.get().size()); - assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records2.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records2.get().get(1).data().asString()); + assertEquals(2, records3.get().size()); + assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records3.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records3.get().get(1).data().asString()); // All rotations for app1 are removed provisionLoadBalancers(app1, clustersPerZone, Collections.emptyMap()); @@ -101,7 +109,7 @@ public class RoutingPolicyMaintainerTest { assertEquals(clustersPerZone * numberOfDeployments, policies.size()); assertTrue("Rotation membership is removed from all policies", policies.stream().allMatch(policy -> policy.rotations().isEmpty())); - assertEquals("Rotations for " + app2 + " are not removed", 2, records2.get().size()); + assertEquals("Rotations for " + app2 + " are not removed", 2, records3.get().size()); } @Test @@ -114,10 +122,10 @@ public class RoutingPolicyMaintainerTest { // Creates records and policies for all clusters in all zones maintainer.maintain(); Set<String> expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(4, policies(app1).size()); @@ -131,12 +139,12 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app1, clustersPerZone + 1); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(6, policies(app1).size()); @@ -146,16 +154,16 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app2, clustersPerZone); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(4, policies(app2).size()); @@ -164,14 +172,14 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app1, clustersPerZone); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); @@ -183,10 +191,10 @@ public class RoutingPolicyMaintainerTest { }); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e60bbdc4d84..f617dc0a447 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1332,7 +1332,8 @@ public class ApplicationApiTest extends ControllerContainerTest { public void applicationWithPerClusterGlobalRotation() { Application app = controllerTester.createApplication(); RoutingPolicy policy = new RoutingPolicy(app.id(), ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ClusterSpec.Id.from("default"), HostName.from("lb-0-canonical-name"), + ClusterSpec.Id.from("default"), controllerTester.controller().system(), + HostName.from("lb-0-canonical-name"), Optional.of("dns-zone-1"), Set.of(RotationName.from("c0"))); tester.controller().curator().writeRoutingPolicies(app.id(), Set.of(policy)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json index cd531bb96da..baaf0cd038d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json @@ -8,7 +8,7 @@ "changeBlockers": [], "compileVersion": "(ignore)", "globalRotations": [ - "https://c0--application1--tenant1.global.vespa.oath.cloud:4443/" + "https://c0.application1.tenant1.global.vespa.oath.cloud/" ], "instances": [], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index f2f38f7f509..b52fec761d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -231,8 +231,6 @@ "changeBlockers": [], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 22e8573b1d4..4b2cb397b5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -219,8 +219,6 @@ ], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 662e045d169..fa903b61825 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -219,8 +219,6 @@ ], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 666c7774cf5..8d1f40260e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -65,8 +65,8 @@ public class RotationRepositoryTest { application = tester.applications().require(application.id()); assertEquals(expected.id(), application.rotation().get()); - assertEquals(URI.create("http://app1.tenant1.global.vespa.yahooapis.com:4080/"), - application.globalDnsName(SystemName.main).get().url()); + assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), + application.endpointsIn(SystemName.main).main().get().url()); try (RotationLock lock = repository.lock()) { Rotation rotation = repository.getOrAssignRotation(tester.applications().require(application.id()), lock); assertEquals(expected, rotation); @@ -153,10 +153,9 @@ public class RotationRepositoryTest { Application application = tester.createApplication("app2", "tenant2", 22L, 2L); tester.deployCompletely(application, applicationPackage); - assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()) - .rotation().get()); - assertEquals("https://cd--app2--tenant2.global.vespa.yahooapis.com:4443/", tester.applications().require(application.id()) - .globalDnsName(SystemName.cd).get().secureUrl().toString()); + assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()).rotation().get()); + assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", tester.applications().require(application.id()) + .endpointsIn(SystemName.cd).main().get().url().toString()); } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java index 574be20e910..1e0d90e71f9 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java @@ -34,7 +34,7 @@ public class BindingRepository<T> implements Iterable<Map.Entry<UriPattern, T>> */ public void bind(String uriPattern, T target) { if (uriPattern.startsWith("https://")) { - log.warning(() -> "Bindings with 'https' scheme are deprecated. Use 'http' to match both 'http' and 'https' in URIs."); + log.warning(() -> String.format("For binding '%s': 'https' is deprecated, use 'http' to match both 'http' and 'https'", uriPattern)); } put(new UriPattern(uriPattern), target); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 5c3298a7aff..9a10c70ceab 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -87,6 +87,14 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog if (clientCert != null && clientCert.length > 0) { accessLogEntry.setSslPrincipal(clientCert[0].getSubjectX500Principal()); } + String sslSessionId = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_SSL_SESSION_ID); + if (sslSessionId != null) { + accessLogEntry.addKeyValue("ssl-session-id", sslSessionId); + } + String cipherSuite = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_CIPHER_SUITE); + if (cipherSuite != null) { + accessLogEntry.addKeyValue("cipher-suite", cipherSuite); + } final long startTime = request.getTimeStamp(); final long endTime = System.currentTimeMillis(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java index 2eb7f432ec2..65c8e153164 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java @@ -39,6 +39,8 @@ public class ServletRequest extends HttpServletRequestWrapper implements Servlet public static final String JDISC_REQUEST_PRINCIPAL = "jdisc.request.principal"; public static final String JDISC_REQUEST_X509CERT = "jdisc.request.X509Certificate"; public static final String SERVLET_REQUEST_X509CERT = "javax.servlet.request.X509Certificate"; + public static final String SERVLET_REQUEST_SSL_SESSION_ID = "javax.servlet.request.ssl_session_id"; + public static final String SERVLET_REQUEST_CIPHER_SUITE = "javax.servlet.request.cipher_suite"; private final HttpServletRequest request; private final HeaderFields headerFields; diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index 288bd6a1eda..e8e752127b1 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -576,7 +576,6 @@ ], "methods": [ "public void <init>()", - "public void addRecurrentTask(com.yahoo.messagebus.Messenger$Task)", "public void start()", "public void deliverMessage(com.yahoo.messagebus.Message, com.yahoo.messagebus.MessageHandler)", "public void deliverReply(com.yahoo.messagebus.Reply, com.yahoo.messagebus.ReplyHandler)", diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 26e61e8917b..0f0b704bba7 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -58,13 +58,13 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, private static Logger log = Logger.getLogger(MessageBus.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final ProtocolRepository protocolRepository = new ProtocolRepository(); - private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<Map<String, RoutingTable>>(null); - private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<String, MessageHandler>(); + private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<>(null); + private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<>(); private final Network net; private final Messenger msn; private final Resender resender; - private int maxPendingCount = 0; - private int maxPendingSize = 0; + private int maxPendingCount; + private int maxPendingSize; private int pendingCount = 0; private int pendingSize = 0; private final Thread careTaker = new Thread(this::sendBlockedMessages); @@ -440,7 +440,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, @Override public void setupRouting(RoutingSpec spec) { - Map<String, RoutingTable> tables = new HashMap<String, RoutingTable>(); + Map<String, RoutingTable> tables = new HashMap<>(); for (int i = 0, len = spec.getNumTables(); i < len; ++i) { RoutingTableSpec table = spec.getTable(i); String name = table.getProtocol(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 4fb83386231..63e5dbb2d04 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -44,7 +44,7 @@ public class Messenger implements Runnable { * * @param task The task to add. */ - public void addRecurrentTask(final Task task) { + void addRecurrentTask(final Task task) { children.add(task); } @@ -159,7 +159,7 @@ public class Messenger implements Runnable { synchronized (this) { if (queue.isEmpty()) { try { - wait(100); + wait(10); } catch (final InterruptedException e) { continue; } @@ -210,13 +210,13 @@ public class Messenger implements Runnable { /** * <p>This method is called when being executed.</p> */ - public void run(); + void run(); /** * <p>This method is called for all tasks, even if {@link #run()} was * never called.</p> */ - public void destroy(); + void destroy(); } private static class MessageTask implements Runnable { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 64b7c4cb12e..e04cccfcbd1 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -1,15 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.routing; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.SendProxy; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.TraceLevel; +import com.yahoo.messagebus.TraceNode; import com.yahoo.messagebus.metrics.RouteMetricSet; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.ServiceAddress; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Stack; import java.util.concurrent.atomic.AtomicInteger; /** @@ -32,7 +45,7 @@ public class RoutingNode implements ReplyHandler { private final AtomicInteger pending = new AtomicInteger(0); private final Message msg; private Reply reply = null; - private Route route = null; + private Route route; private RoutingPolicy policy = null; private RoutingContext routingContext = null; private ServiceAddress serviceAddress = null; @@ -122,7 +135,7 @@ public class RoutingNode implements ReplyHandler { * * @param msg The error message to assign. */ - public void notifyAbort(String msg) { + private void notifyAbort(String msg) { Stack<RoutingNode> stack = new Stack<>(); stack.push(this); while (!stack.isEmpty()) { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java index 0ac962ad947..2ebe4103713 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java @@ -4,8 +4,17 @@ package com.yahoo.messagebus.routing; import com.yahoo.component.Vtag; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; @@ -23,7 +32,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 10ac30d8715..1811fc0c8f0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import java.net.URI; @@ -16,12 +17,12 @@ import java.util.stream.Collectors; */ public class ConfigServerInfo { private final URI loadBalancerEndpoint; - private final AthenzService configServerIdentity; + private final AthenzIdentity configServerIdentity; private final Function<String, URI> configServerHostnameToUriMapper; private final List<URI> configServerURIs; public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, - String scheme, int port, AthenzService configServerAthenzIdentity) { + String scheme, int port, AthenzIdentity configServerAthenzIdentity) { this.loadBalancerEndpoint = createLoadBalancerEndpoint(loadBalancerHostName, scheme, port); this.configServerIdentity = configServerAthenzIdentity; this.configServerHostnameToUriMapper = hostname -> URI.create(scheme + "://" + hostname + ":" + port); @@ -46,7 +47,7 @@ public class ConfigServerInfo { return loadBalancerEndpoint; } - public AthenzService getConfigServerIdentity() { + public AthenzIdentity getConfigServerIdentity() { return configServerIdentity; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 4fe0f420f05..550d6e7021e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -8,6 +8,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; @@ -63,7 +64,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final URI ztsEndpoint; private final Path trustStorePath; - private final AthenzService configserverIdentity; + private final AthenzIdentity configserverIdentity; private final Clock clock; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index 496f4bd667d..205e7b1e258 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; @@ -33,7 +34,7 @@ public interface NodeAgentContext extends TaskContext { return node().getNodeType(); } - AthenzService identity(); + AthenzIdentity identity(); DockerNetworking dockerNetworking(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 9ca19a76706..1b33fed151e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -4,6 +4,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; @@ -30,7 +31,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final NodeSpec node; private final Acl acl; private final ContainerName containerName; - private final AthenzService identity; + private final AthenzIdentity identity; private final DockerNetworking dockerNetworking; private final ZoneId zoneId; private final Path pathToNodeRootOnHost; @@ -38,7 +39,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final String vespaUser; private final String vespaUserOnHost; - public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzService identity, + public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, DockerNetworking dockerNetworking, ZoneId zoneId, Path pathToContainerStorage, Path pathToVespaHome, String vespaUser, String vespaUserOnHost) { @@ -71,7 +72,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override - public AthenzService identity() { + public AthenzIdentity identity() { return identity; } @@ -157,7 +158,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { public static class Builder { private NodeSpec.Builder nodeSpecBuilder = new NodeSpec.Builder(); private Acl acl; - private AthenzService identity; + private AthenzIdentity identity; private DockerNetworking dockerNetworking; private ZoneId zoneId; private Path pathToContainerStorage; @@ -192,7 +193,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } - public Builder identity(AthenzService identity) { + public Builder identity(AthenzIdentity identity) { this.identity = identity; return this; } diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 18d4fbdb6d7..b2334ed025e 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -5,7 +5,7 @@ LOG_SETUP("feed_and_search_test"); #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/fef.h> diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index be41aa96efd..9a1ddee4278 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/searchcorespi/index/fusionrunner.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/matchdatalayout.h> diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 82a3590b6b9..841c69289bf 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -14,11 +14,10 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> @@ -43,7 +42,7 @@ using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::schema::DataType; using vespalib::makeLambdaTask; -using search::memoryindex::CompactDocumentWordsStore; +using search::memoryindex::CompactWordsStore; using search::memoryindex::FieldIndexCollection; using search::queryeval::Source; using std::set; diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 6124eadb34d..2ca6f969c55 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchcorespi/index/imemoryindex.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/tunefileinfo.h> diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 6d1695155a8..66408b1d7d7 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -173,14 +173,14 @@ vespa_define_module( src/tests/index/doctypebuilder src/tests/indexmetainfo src/tests/ld-library-path - src/tests/memoryindex/compact_document_words_store + src/tests/memoryindex/compact_words_store src/tests/memoryindex/datastore - src/tests/memoryindex/document_remover - src/tests/memoryindex/documentinverter + src/tests/memoryindex/document_inverter src/tests/memoryindex/field_index - src/tests/memoryindex/fieldinverter - src/tests/memoryindex/memoryindex - src/tests/memoryindex/urlfieldinverter + src/tests/memoryindex/field_index_remover + src/tests/memoryindex/field_inverter + src/tests/memoryindex/memory_index + src/tests/memoryindex/url_field_inverter src/tests/nativerank src/tests/nearsearch src/tests/postinglistbm diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index b911284a1b4..2ef8448db8b 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index fb6535c4a70..24146e516a0 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -12,9 +12,9 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore deleted file mode 100644 index 3ad290f1731..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_compact_document_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt deleted file mode 100644 index a5c8e0b2d14..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_compact_document_words_store_test_app TEST - SOURCES - compact_document_words_store_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_compact_document_words_store_test_app COMMAND searchlib_compact_document_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore new file mode 100644 index 00000000000..9f9acb50adc --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore @@ -0,0 +1 @@ +searchlib_compact_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt new file mode 100644 index 00000000000..ee31ef7c7aa --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_compact_words_store_test_app TEST + SOURCES + compact_words_store_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_compact_words_store_test_app COMMAND searchlib_compact_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp index 6e22a4e5ff6..04d66396f90 100644 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp +++ b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp @@ -2,7 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/datastore/entryref.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <iostream> @@ -12,8 +12,8 @@ using namespace search; using namespace search::datastore; using namespace search::memoryindex; -typedef CompactDocumentWordsStore::Builder Builder; -typedef CompactDocumentWordsStore::Iterator Iterator; +typedef CompactWordsStore::Builder Builder; +typedef CompactWordsStore::Iterator Iterator; typedef Builder::WordRefVector WordRefVector; const EntryRef w1(1); @@ -53,7 +53,7 @@ toStr(Iterator itr) struct SingleFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; SingleFixture() : _store() { _store.insert(Builder(d1).insert(w1).insert(w2).insert(w3)); } @@ -61,7 +61,7 @@ struct SingleFixture struct MultiFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; MultiFixture() : _store() { _store.insert(Builder(d1).insert(w1)); _store.insert(Builder(d2).insert(w2)); @@ -100,7 +100,7 @@ TEST_F("require that documents can be removed and re-inserted", MultiFixture) TEST("require that a lot of words can be inserted, retrieved and removed") { - CompactDocumentWordsStore store; + CompactWordsStore store; for (uint32_t docId = 0; docId < 50; ++docId) { Builder b(docId); for (uint32_t wordRef = 0; wordRef < 20000; ++wordRef) { @@ -125,9 +125,9 @@ TEST("require that a lot of words can be inserted, retrieved and removed") TEST("require that initial memory usage is reported") { - CompactDocumentWordsStore store; - CompactDocumentWordsStore::DocumentWordsMap docs; - CompactDocumentWordsStore::Store internalStore; + CompactWordsStore store; + CompactWordsStore::DocumentWordsMap docs; + CompactWordsStore::Store internalStore; MemoryUsage initExp; initExp.incAllocatedBytes(docs.getMemoryConsumption()); initExp.incUsedBytes(docs.getMemoryUsed()); @@ -142,7 +142,7 @@ TEST("require that initial memory usage is reported") TEST("require that memory usage is updated after insert") { - CompactDocumentWordsStore store; + CompactWordsStore store; MemoryUsage init = store.getMemoryUsage(); store.insert(Builder(d1).insert(w1)); diff --git a/searchlib/src/tests/memoryindex/datastore/.gitignore b/searchlib/src/tests/memoryindex/datastore/.gitignore index 98f4acc70a8..a5842a0fd69 100644 --- a/searchlib/src/tests/memoryindex/datastore/.gitignore +++ b/searchlib/src/tests/memoryindex/datastore/.gitignore @@ -1,8 +1,4 @@ .depend Makefile -datastore_test -featurestore_test -wordstore_test -searchlib_datastore_test_app -searchlib_featurestore_test_app -searchlib_wordstore_test_app +searchlib_feature_store_test_app +searchlib_word_store_test_app diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt index 2ba0f2eac63..45507f3b0ae 100644 --- a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -1,15 +1,15 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_featurestore_test_app TEST +vespa_add_executable(searchlib_feature_store_test_app TEST SOURCES - featurestore_test.cpp + feature_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_featurestore_test_app COMMAND searchlib_featurestore_test_app) -vespa_add_executable(searchlib_wordstore_test_app TEST +vespa_add_test(NAME searchlib_feature_store_test_app COMMAND searchlib_feature_store_test_app) +vespa_add_executable(searchlib_word_store_test_app TEST SOURCES - wordstore_test.cpp + word_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_wordstore_test_app COMMAND searchlib_wordstore_test_app) +vespa_add_test(NAME searchlib_word_store_test_app COMMAND searchlib_word_store_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index dc061f55732..49e9d613861 100644 --- a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("featurestore_test"); +LOG_SETUP("feature_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> using namespace search::btree; using namespace search::datastore; @@ -213,7 +213,7 @@ Test::Test() int Test::Main() { - TEST_INIT("featurestore_test"); + TEST_INIT("feature_store_test"); requireThatFeaturesCanBeAddedAndRetrieved(); requireThatNextWordsAreWorking(); diff --git a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp index c1baff72514..b7f454bfdf7 100644 --- a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("wordstore_test"); +LOG_SETUP("word_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> using namespace search::datastore; @@ -75,7 +75,7 @@ Test::requireThatAddWordTriggersChangeOfBuffer() int Test::Main() { - TEST_INIT("wordstore_test"); + TEST_INIT("word_store_test"); requireThatWordsCanBeAddedAndRetrieved(); requireThatAddWordTriggersChangeOfBuffer(); diff --git a/searchlib/src/tests/memoryindex/document_inverter/.gitignore b/searchlib/src/tests/memoryindex/document_inverter/.gitignore new file mode 100644 index 00000000000..6245bb146e7 --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_document_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt new file mode 100644 index 00000000000..1058a19d0ce --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_document_inverter_test_app TEST + SOURCES + document_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_document_inverter_test_app COMMAND searchlib_document_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index 36cd15c8ada..91c1ccba706 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -3,9 +3,9 @@ #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> @@ -97,7 +97,7 @@ struct Fixture SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; DocumentInverter _inv; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt deleted file mode 100644 index 04e5b0ec126..00000000000 --- a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_document_remover_test_app TEST - SOURCES - document_remover_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_document_remover_test_app COMMAND searchlib_document_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/.gitignore b/searchlib/src/tests/memoryindex/documentinverter/.gitignore deleted file mode 100644 index 1e9666b2d63..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_documentinverter_test_app diff --git a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt deleted file mode 100644 index c7760e9f12f..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_documentinverter_test_app TEST - SOURCES - documentinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_documentinverter_test_app COMMAND searchlib_documentinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 373ed7fd311..3a635756ec7 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -10,11 +10,11 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/vespalib/testkit/testapp.h> @@ -300,7 +300,7 @@ MockFieldIndex::~MockFieldIndex() = default; /** * MockWordStoreScan is a helper class to ensure that previous word is - * still stored safely in memory, to satisfy OrderedDocumentInserter + * still stored safely in memory, to satisfy OrderedFieldIndexInserter * needs. */ class MockWordStoreScan @@ -347,7 +347,7 @@ class MyInserter MockFieldIndex _mock; FieldIndexCollection _fieldIndexes; DocIdAndPosOccFeatures _features; - IOrderedDocumentInserter *_documentInserter; + IOrderedFieldIndexInserter *_inserter; public: MyInserter(const Schema &schema) @@ -355,7 +355,7 @@ public: _mock(), _fieldIndexes(schema), _features(), - _documentInserter(nullptr) + _inserter(nullptr) { _features.addNextOcc(0, 0, 1, 1); } @@ -365,32 +365,32 @@ public: setNextWord(const vespalib::string &word) { const vespalib::string &w = _wordStoreScan.setWord(word); - _documentInserter->setNextWord(w); + _inserter->setNextWord(w); _mock.setNextWord(w); } void setNextField(uint32_t fieldId) { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } - _documentInserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); - _documentInserter->rewind(); + _inserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); + _inserter->rewind(); _mock.setNextField(fieldId); } void add(uint32_t docId) { - _documentInserter->add(docId, _features); + _inserter->add(docId, _features); _mock.add(docId); } void remove(uint32_t docId) { - _documentInserter->remove(docId); + _inserter->remove(docId); _mock.remove(docId); } @@ -406,8 +406,8 @@ public: bool assertPostings() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } for (auto wfp : _mock) { auto &wf = wfp.first; @@ -423,9 +423,9 @@ public: void rewind() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); - _documentInserter = nullptr; + if (_inserter != nullptr) { + _inserter->flush(); + _inserter = nullptr; } } @@ -451,7 +451,7 @@ myremove(uint32_t docId, DocumentInverter &inv, FieldIndexCollection &fieldIndex class WrapInserter { - OrderedDocumentInserter &_inserter; + OrderedFieldIndexInserter &_inserter; public: WrapInserter(FieldIndexCollection &fieldIndexes, uint32_t fieldId) : _inserter(fieldIndexes.getFieldIndex(fieldId)->getInserter()) @@ -503,9 +503,9 @@ public: }; -class MyDrainRemoves : IDocumentRemoveListener +class MyDrainRemoves : IFieldIndexRemoveListener { - DocumentRemover &_remover; + FieldIndexRemover &_remover; public: virtual void remove(const vespalib::stringref, uint32_t) override { } diff --git a/searchlib/src/tests/memoryindex/document_remover/.gitignore b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore index 2126f9147bd..2126f9147bd 100644 --- a/searchlib/src/tests/memoryindex/document_remover/.gitignore +++ b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore diff --git a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt new file mode 100644 index 00000000000..ef75337c6b6 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_index_remover_test_app TEST + SOURCES + field_index_remover_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_field_index_remover_test_app COMMAND searchlib_field_index_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp index af7a9422e49..fed6d963b70 100644 --- a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp @@ -2,9 +2,9 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/document_remover.h> -#include <vespa/searchlib/memoryindex/wordstore.h> -#include <vespa/searchlib/memoryindex/i_document_remove_listener.h> +#include <vespa/searchlib/memoryindex/field_index_remover.h> +#include <vespa/searchlib/memoryindex/i_field_index_remove_listener.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/test/insertion_operators.h> #include <algorithm> @@ -38,7 +38,7 @@ operator<<(std::ostream &os, const WordFieldPair &val) return os; } -struct MockRemoveListener : public IDocumentRemoveListener +struct MockRemoveListener : public IFieldIndexRemoveListener { WordFieldVector _words; uint32_t _expDocId; @@ -65,7 +65,7 @@ struct Fixture MockRemoveListener _listener; std::vector<std::unique_ptr<WordStore>> _wordStores; std::vector<std::map<vespalib::string, datastore::EntryRef>> _wordToRefMaps; - std::vector<std::unique_ptr<DocumentRemover>> _removers; + std::vector<std::unique_ptr<FieldIndexRemover>> _removers; Fixture() : _listener(), _wordStores(), @@ -75,7 +75,7 @@ struct Fixture uint32_t numFields = 4; for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { _wordStores.push_back(std::make_unique<WordStore>()); - _removers.push_back(std::make_unique<DocumentRemover> + _removers.push_back(std::make_unique<FieldIndexRemover> (*_wordStores.back())); } _wordToRefMaps.resize(numFields); diff --git a/searchlib/src/tests/memoryindex/field_inverter/.gitignore b/searchlib/src/tests/memoryindex/field_inverter/.gitignore new file mode 100644 index 00000000000..58579d09421 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..f39e05d6823 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_inverter_test_app TEST + SOURCES + field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_field_inverter_test_app COMMAND searchlib_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp index 1d066747ef8..f08e61b0da2 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -105,7 +105,7 @@ struct Fixture Schema _schema; DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore b/searchlib/src/tests/memoryindex/fieldinverter/.gitignore deleted file mode 100644 index 482663dd92e..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_fieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt deleted file mode 100644 index b6b223dceed..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_fieldinverter_test_app TEST - SOURCES - fieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_fieldinverter_test_app COMMAND searchlib_fieldinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/memory_index/.gitignore b/searchlib/src/tests/memoryindex/memory_index/.gitignore new file mode 100644 index 00000000000..6ec9ccf5015 --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +searchlib_memory_index_test_app diff --git a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt new file mode 100644 index 00000000000..d6b6691f05d --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_memory_index_test_app TEST + SOURCES + memory_index_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_memory_index_test_app COMMAND searchlib_memory_index_test_app) diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 9de6ac9f310..50f44074683 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -2,24 +2,24 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> -LOG_SETUP("memoryindex_test"); +LOG_SETUP("memory_index_test"); using document::Document; using document::FieldValue; diff --git a/searchlib/src/tests/memoryindex/memoryindex/.gitignore b/searchlib/src/tests/memoryindex/memoryindex/.gitignore deleted file mode 100644 index 174d0a494e2..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -memoryindex_test -sourceselectorwriter_test -searchlib_memoryindex_test_app diff --git a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt b/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt deleted file mode 100644 index 20f526e5c99..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_memoryindex_test_app TEST - SOURCES - memoryindex_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_memoryindex_test_app COMMAND searchlib_memoryindex_test_app) diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore new file mode 100644 index 00000000000..694dc947042 --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_url_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..28efc8a861e --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_url_field_inverter_test_app TEST + SOURCES + url_field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_url_field_inverter_test_app COMMAND searchlib_url_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp index daec09828f6..76fbf662b77 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* -*- mode: C++; coding: utf-8; -*- */ - +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/urlfieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/url_field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -183,7 +182,7 @@ struct Fixture DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; std::unique_ptr<UrlFieldInverter> _urlInverter; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; index::SchemaIndexFields _schemaIndexFields; static Schema diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore b/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore deleted file mode 100644 index b2636fe5e81..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_urlfieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt deleted file mode 100644 index 16fa8f5952e..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_urlfieldinverter_test_app TEST - SOURCES - urlfieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_urlfieldinverter_test_app COMMAND searchlib_urlfieldinverter_test_app) diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.h b/searchlib/src/vespa/searchlib/index/docidandfeatures.h index 91a500495cc..a9329c9fa01 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.h +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.h @@ -183,14 +183,6 @@ public: void setRaw(bool raw) { _raw = raw; } bool getRaw() const { return _raw; } - - /** - * Append features from a single field to a field collection. - * - * @param rhs features for a single field - * @param localFieldId local field id for the field - */ - void append(const DocIdAndFeatures &rhs, uint32_t localFieldId); }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index ffcd7ebd975..441fe12c383 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,17 +1,17 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_memoryindex OBJECT SOURCES - compact_document_words_store.cpp - documentinverter.cpp - document_remover.cpp - featurestore.cpp + compact_words_store.cpp + document_inverter.cpp + feature_store.cpp field_index.cpp field_index_collection.cpp - fieldinverter.cpp - memoryindex.cpp - ordereddocumentinserter.cpp - postingiterator.cpp - urlfieldinverter.cpp - wordstore.cpp + field_index_remover.cpp + field_inverter.cpp + memory_index.cpp + ordered_field_index_inserter.cpp + posting_iterator.cpp + url_field_inverter.cpp + word_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp index e2d089626b1..27282282c11 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp @@ -1,15 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compact_document_words_store.h" +#include "compact_words_store.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.compact_document_words_store"); +LOG_SETUP(".memoryindex.compact_words_store"); namespace search::memoryindex { -using Builder = CompactDocumentWordsStore::Builder; +using Builder = CompactWordsStore::Builder; namespace { @@ -36,28 +36,28 @@ serialize(const Builder &builder, uint32_t *begin) } -CompactDocumentWordsStore::Builder::Builder(uint32_t docId_) +CompactWordsStore::Builder::Builder(uint32_t docId_) : _docId(docId_), _words() { } -CompactDocumentWordsStore::Builder::~Builder() { } +CompactWordsStore::Builder::~Builder() { } -CompactDocumentWordsStore::Builder & -CompactDocumentWordsStore::Builder::insert(datastore::EntryRef wordRef) +CompactWordsStore::Builder & +CompactWordsStore::Builder::insert(datastore::EntryRef wordRef) { _words.push_back(wordRef); return *this; } inline void -CompactDocumentWordsStore::Iterator::nextWord() +CompactWordsStore::Iterator::nextWord() { _wordRef = *_buf++; _remainingWords--; } -CompactDocumentWordsStore::Iterator::Iterator() +CompactWordsStore::Iterator::Iterator() : _buf(nullptr), _remainingWords(0), _wordRef(0), @@ -65,7 +65,7 @@ CompactDocumentWordsStore::Iterator::Iterator() { } -CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) +CompactWordsStore::Iterator::Iterator(const uint32_t *buf) : _buf(buf), _remainingWords(0), _wordRef(0), @@ -79,8 +79,8 @@ CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) } } -CompactDocumentWordsStore::Iterator & -CompactDocumentWordsStore::Iterator::operator++() +CompactWordsStore::Iterator & +CompactWordsStore::Iterator::operator++() { if (_remainingWords > 0) { nextWord(); @@ -90,7 +90,7 @@ CompactDocumentWordsStore::Iterator::operator++() return *this; } -CompactDocumentWordsStore::Store::Store() +CompactWordsStore::Store::Store() : _store(), _type(1, MIN_BUFFER_ARRAYS, @@ -101,13 +101,13 @@ CompactDocumentWordsStore::Store::Store() _store.initActiveBuffers(); } -CompactDocumentWordsStore::Store::~Store() +CompactWordsStore::Store::~Store() { _store.dropBuffers(); } datastore::EntryRef -CompactDocumentWordsStore::Store::insert(const Builder &builder) +CompactWordsStore::Store::insert(const Builder &builder) { size_t serializedSize = getSerializedSize(builder); auto result = _store.rawAllocator<uint32_t>(_typeId).alloc(serializedSize); @@ -118,26 +118,26 @@ CompactDocumentWordsStore::Store::insert(const Builder &builder) return result.ref; } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::Store::get(datastore::EntryRef ref) const +CompactWordsStore::Iterator +CompactWordsStore::Store::get(datastore::EntryRef wordRef) const { - RefType internalRef(ref); + RefType internalRef(wordRef); const uint32_t *buf = _store.getEntry<uint32_t>(internalRef); return Iterator(buf); } -CompactDocumentWordsStore::CompactDocumentWordsStore() +CompactWordsStore::CompactWordsStore() : _docs(), _wordsStore() { } -CompactDocumentWordsStore::~CompactDocumentWordsStore() { } +CompactWordsStore::~CompactWordsStore() { } void -CompactDocumentWordsStore::insert(const Builder &builder) +CompactWordsStore::insert(const Builder &builder) { - datastore::EntryRef ref = _wordsStore.insert(builder); - auto insres = _docs.insert(std::make_pair(builder.docId(), ref)); + datastore::EntryRef wordRef = _wordsStore.insert(builder); + auto insres = _docs.insert(std::make_pair(builder.docId(), wordRef)); if (!insres.second) { LOG(error, "Failed inserting remove info for docid %u", builder.docId()); @@ -146,13 +146,13 @@ CompactDocumentWordsStore::insert(const Builder &builder) } void -CompactDocumentWordsStore::remove(uint32_t docId) +CompactWordsStore::remove(uint32_t docId) { _docs.erase(docId); } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::get(uint32_t docId) const +CompactWordsStore::Iterator +CompactWordsStore::get(uint32_t docId) const { auto itr = _docs.find(docId); if (itr != _docs.end()) { @@ -162,7 +162,7 @@ CompactDocumentWordsStore::get(uint32_t docId) const } MemoryUsage -CompactDocumentWordsStore::getMemoryUsage() const +CompactWordsStore::getMemoryUsage() const { MemoryUsage usage; usage.incAllocatedBytes(_docs.getMemoryConsumption()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h index ced7ec241bd..2fc6ec8d5bb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h @@ -10,18 +10,16 @@ namespace search::memoryindex { /** - * Class used to store the {wordRef, fieldId, docId} tuples that are inserted - * into the memory index dictionary. These tuples are later used when removing - * all remains of a document from the posting lists of the dictionary. + * Class used to store the {wordRef, docId} tuples that are inserted into a FieldIndex and its posting lists. + * + * These tuples are later used when removing all remains of a document from the posting lists in that index. */ -class CompactDocumentWordsStore -{ +class CompactWordsStore { public: /** - * Builder used to collect all wordRefs for a field. + * Builder used to collect all words (as wordRefs) for a docId in a field. */ - class Builder - { + class Builder { public: using UP = std::unique_ptr<Builder>; using WordRefVector = vespalib::Array<datastore::EntryRef>; @@ -39,10 +37,9 @@ public: }; /** - * Iterator over all {wordRef, fieldId} pairs for a document. + * Iterator over all words (as wordRefs) for a docId in a field. */ - class Iterator - { + class Iterator { private: const uint32_t *_buf; uint32_t _remainingWords; @@ -61,10 +58,9 @@ public: }; /** - * Store for all {wordRef, fieldId} pairs among all documents. + * Store for all unique words (as wordRefs) among all documents. */ - class Store - { + class Store { public: using DataStoreType = datastore::DataStoreT<datastore::EntryRefT<22>>; using RefType = DataStoreType::RefType; @@ -78,7 +74,7 @@ public: Store(); ~Store(); datastore::EntryRef insert(const Builder &builder); - Iterator get(datastore::EntryRef ref) const; + Iterator get(datastore::EntryRef wordRef) const; MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } }; @@ -89,8 +85,8 @@ private: Store _wordsStore; public: - CompactDocumentWordsStore(); - ~CompactDocumentWordsStore(); + CompactWordsStore(); + ~CompactWordsStore(); void insert(const Builder &builder); void remove(uint32_t docId); Iterator get(uint32_t docId) const; diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 1501ff7d2fc..a468428e21f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" +#include "document_inverter.h" #include "field_index_collection.h" -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" -#include "urlfieldinverter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include "url_field_inverter.h" #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/repo/fixedtyperepo.h> @@ -16,7 +16,7 @@ #include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.documentinverter"); +LOG_SETUP(".memoryindex.document_inverter"); namespace search::memoryindex { @@ -39,7 +39,6 @@ using index::DocIdAndPosOccFeatures; using index::Schema; using search::util::URL; - DocumentInverter::DocumentInverter(const Schema &schema, ISequencedTaskExecutor &invertThreads, ISequencedTaskExecutor &pushThreads) @@ -74,14 +73,12 @@ DocumentInverter::DocumentInverter(const Schema &schema, } } - DocumentInverter::~DocumentInverter() { _invertThreads.sync(); _pushThreads.sync(); } - void DocumentInverter::addFieldPath(const document::DocumentType &docType, uint32_t fieldId) @@ -100,9 +97,9 @@ DocumentInverter::addFieldPath(const document::DocumentType &docType, _indexedFieldPaths[fieldId] = std::move(fp); } - -void DocumentInverter::buildFieldPath(const document::DocumentType &docType, - const document::DataType *dataType) +void +DocumentInverter::buildFieldPath(const document::DocumentType &docType, + const document::DataType *dataType) { _indexedFieldPaths.clear(); _indexedFieldPaths.resize(_schema.getNumIndexFields()); @@ -115,7 +112,6 @@ void DocumentInverter::buildFieldPath(const document::DocumentType &docType, _dataType = dataType; } - void DocumentInverter::invertDocument(uint32_t docId, const Document &doc) { @@ -154,7 +150,6 @@ DocumentInverter::invertDocument(uint32_t docId, const Document &doc) } } - void DocumentInverter::removeDocument(uint32_t docId) { @@ -175,7 +170,6 @@ DocumentInverter::removeDocument(uint32_t docId) } } - void DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, const std::shared_ptr<IDestructorCallback> &onWriteDone) @@ -184,8 +178,8 @@ DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, uint32_t fieldId = 0; for (auto &inverter : _inverters) { FieldIndex &fieldIndex(**indexFieldIterator); - DocumentRemover &remover(fieldIndex.getDocumentRemover()); - OrderedDocumentInserter &inserter(fieldIndex.getInserter()); + FieldIndexRemover &remover(fieldIndex.getDocumentRemover()); + OrderedFieldIndexInserter &inserter(fieldIndex.getInserter()); _pushThreads.execute(fieldId, [inverter(inverter.get()), &remover, &inserter, &fieldIndex, onWriteDone]() diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index fa8d13d98fc..5c2d9cc84ed 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -2,7 +2,7 @@ #pragma once -#include "i_document_remove_listener.h" +#include "i_field_index_remove_listener.h" #include <vespa/searchlib/index/schema_index_fields.h> namespace document { @@ -24,8 +24,7 @@ class FieldInverter; class UrlFieldInverter; class FieldIndexCollection; -class DocumentInverter -{ +class DocumentInverter { private: DocumentInverter(const DocumentInverter &) = delete; DocumentInverter &operator=(const DocumentInverter &) = delete; diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index c032bb33217..974fcc01c36 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "featurestore.h" +#include "feature_store.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/datastore/datastore.hpp> @@ -26,7 +26,6 @@ FeatureStore::writeFeatures(uint32_t packedIndex, const DocIdAndFeatures &featur return oldOffset; } - datastore::EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { @@ -43,7 +42,6 @@ FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) return result.ref; } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) { @@ -58,7 +56,6 @@ FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) return std::make_pair(ref, bitLen); } - datastore::EntryRef FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) { @@ -70,7 +67,6 @@ FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) return newRef; } - FeatureStore::FeatureStore(const Schema &schema) : _store(), _f(nullptr), @@ -95,13 +91,11 @@ FeatureStore::FeatureStore(const Schema &schema) _store.initActiveBuffers(); } - FeatureStore::~FeatureStore() { _store.dropBuffers(); } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features) { @@ -111,8 +105,6 @@ FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features return addFeatures(oldOffset, newOffset); } - - void FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAndFeatures &features) { @@ -121,7 +113,6 @@ FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAn _d.readFeatures(features); } - size_t FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) { @@ -135,7 +126,6 @@ FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) return bitLen; } - datastore::EntryRef FeatureStore::moveFeatures(uint32_t packedIndex, datastore::EntryRef ref) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index ef75b9f6d31..94d44eaf44d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -9,8 +9,7 @@ namespace search::memoryindex { -class FeatureStore -{ +class FeatureStore { public: using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; using RefType = DataStoreType::RefType; @@ -122,9 +121,7 @@ public: * @param packedIndex The field or field collection owning features * @param decoder The feature decoder */ - void - setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const - { + void setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const { decoder._fieldsParams = &_fieldsParams[packedIndex]; } @@ -135,9 +132,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { const uint8_t * bits = getBits(ref); decoder.setByteCompr(bits); uint32_t bufferId = RefType(ref).bufferId(); @@ -155,9 +150,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { decoder.setByteCompr(getBits(ref)); } @@ -169,8 +162,7 @@ public: * @param ref Reference to stored features * @return size of features in bits */ - size_t - bitSize(uint32_t packedIndex, datastore::EntryRef ref); + size_t bitSize(uint32_t packedIndex, datastore::EntryRef ref); /** * Get byte address of stored features diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 4d42b9ae493..7d10895c32f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_index.h" -#include "ordereddocumentinserter.h" +#include "ordered_field_index_inserter.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -38,7 +38,7 @@ FieldIndex::FieldIndex(const Schema & schema, uint32_t fieldId) _featureStore(schema), _fieldId(fieldId), _remover(_wordStore), - _inserter(std::make_unique<OrderedDocumentInserter>(*this)) + _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)) { } FieldIndex::~FieldIndex() @@ -88,7 +88,6 @@ FieldIndex::findFrozen(const vespalib::stringref word) const return PostingList::Iterator(); } - void FieldIndex::compactFeatures() { @@ -218,7 +217,6 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) } } - MemoryUsage FieldIndex::getMemoryUsage() const { @@ -231,7 +229,7 @@ FieldIndex::getMemoryUsage() const return usage; } -} // namespace search::memoryindex +} namespace search::btree { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index 4a27e30b47a..3b0675b5fdf 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -2,12 +2,12 @@ #pragma once -#include "featurestore.h" -#include "wordstore.h" -#include "document_remover.h" -#include <vespa/searchlib/btree/btreeroot.h> +#include "feature_store.h" +#include "field_index_remover.h" +#include "word_store.h" #include <vespa/searchlib/btree/btree.h> #include <vespa/searchlib/btree/btreenodeallocator.h> +#include <vespa/searchlib/btree/btreeroot.h> #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/indexbuilder.h> @@ -16,7 +16,7 @@ namespace search::memoryindex { -class OrderedDocumentInserter; +class OrderedFieldIndexInserter; /** * Memory index for a single field. @@ -56,9 +56,7 @@ public: const WordStore &_wordStore; const vespalib::stringref _word; - const char * - getWord(datastore::EntryRef wordRef) const - { + const char *getWord(datastore::EntryRef wordRef) const { if (wordRef.valid()) { return _wordStore.getWord(wordRef); } @@ -71,9 +69,7 @@ public: _word(word) { } - bool - operator()(const WordKey & lhs, const WordKey & rhs) const - { + bool operator()(const WordKey & lhs, const WordKey & rhs) const { int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef)); return cmpres < 0; } @@ -93,8 +89,8 @@ private: PostingListStore _postingListStore; FeatureStore _featureStore; uint32_t _fieldId; - DocumentRemover _remover; - std::unique_ptr<OrderedDocumentInserter> _inserter; + FieldIndexRemover _remover; + std::unique_ptr<OrderedFieldIndexInserter> _inserter; public: datastore::EntryRef addWord(const vespalib::stringref word) { @@ -102,9 +98,7 @@ public: return _wordStore.addWord(word); } - datastore::EntryRef - addFeatures(const index::DocIdAndFeatures &features) - { + datastore::EntryRef addFeatures(const index::DocIdAndFeatures &features) { return _featureStore.addFeatures(_fieldId, features).first; } @@ -118,7 +112,7 @@ public: uint64_t getNumUniqueWords() const { return _numUniqueWords; } const FeatureStore & getFeatureStore() const { return _featureStore; } const WordStore &getWordStore() const { return _wordStore; } - OrderedDocumentInserter &getInserter() const { return *_inserter; } + OrderedFieldIndexInserter &getInserter() const { return *_inserter; } private: void freeze() { @@ -126,9 +120,7 @@ private: _dict.getAllocator().freeze(); } - void - trimHoldLists() - { + void trimHoldLists() { GenerationHandler::generation_t usedGen = _generationHandler.getFirstUsedGeneration(); _postingListStore.trimHoldLists(usedGen); @@ -136,9 +128,7 @@ private: _featureStore.trimHoldLists(usedGen); } - void - transferHoldLists() - { + void transferHoldLists() { GenerationHandler::generation_t generation = _generationHandler.getCurrentGeneration(); _postingListStore.transferHoldLists(generation); @@ -146,9 +136,7 @@ private: _featureStore.transferHoldLists(generation); } - void - incGeneration() - { + void incGeneration() { _generationHandler.incGeneration(); } @@ -163,27 +151,11 @@ public: void dump(search::index::IndexBuilder & indexBuilder); MemoryUsage getMemoryUsage() const; + DictionaryTree &getDictionaryTree() { return _dict; } + PostingListStore &getPostingListStore() { return _postingListStore; } + FieldIndexRemover &getDocumentRemover() { return _remover; } - DictionaryTree & - getDictionaryTree() - { - return _dict; - } - - PostingListStore & - getPostingListStore() - { - return _postingListStore; - } - - DocumentRemover & - getDocumentRemover() - { - return _remover; - } - - void commit() - { + void commit() { _remover.flush(); freeze(); transferHoldLists(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index 45431f0e8ef..27944b5fe89 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_index_collection.h" -#include "fieldinverter.h" +#include "field_inverter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/btree/btreenode.hpp> @@ -40,7 +40,6 @@ FieldIndexCollection::~FieldIndexCollection() { } - void FieldIndexCollection::dump(search::index::IndexBuilder &indexBuilder) { @@ -61,6 +60,5 @@ FieldIndexCollection::getMemoryUsage() const return usage; } - } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index 3b8e63626bf..5c2aa6f9b2c 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -6,7 +6,7 @@ namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class FieldInverter; /** @@ -29,14 +29,11 @@ public: FieldIndexCollection(const index::Schema &schema); ~FieldIndexCollection(); PostingList::Iterator find(const vespalib::stringref word, - uint32_t fieldId) const - { + uint32_t fieldId) const { return _fieldIndexes[fieldId]->find(word); } - PostingList::ConstIterator - findFrozen(const vespalib::stringref word, uint32_t fieldId) const - { + PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t fieldId) const { return _fieldIndexes[fieldId]->findFrozen(word); } @@ -56,8 +53,7 @@ public: return _fieldIndexes[fieldId].get(); } - const std::vector<std::unique_ptr<FieldIndex>> & - getFieldIndexes() const { return _fieldIndexes; } + const std::vector<std::unique_ptr<FieldIndex>> &getFieldIndexes() const { return _fieldIndexes; } uint32_t getNumFields() const { return _numFields; } }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp index 67b519bbadc..2afddf072f2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "document_remover.h" -#include "i_document_remove_listener.h" -#include "wordstore.h" + +#include "field_index_remover.h" +#include "i_field_index_remove_listener.h" +#include "word_store.h" #include <vespa/searchlib/common/sort.h> namespace search::memoryindex { -using Builder = CompactDocumentWordsStore::Builder; -using Iterator = CompactDocumentWordsStore::Iterator; +using Builder = CompactWordsStore::Builder; +using Iterator = CompactWordsStore::Iterator; -DocumentRemover::DocumentRemover(const WordStore &wordStore) +FieldIndexRemover::FieldIndexRemover(const WordStore &wordStore) : _store(), _builder(), _wordFieldDocTuples(), @@ -17,11 +18,10 @@ DocumentRemover::DocumentRemover(const WordStore &wordStore) { } -DocumentRemover::~DocumentRemover() { -} +FieldIndexRemover::~FieldIndexRemover() = default; void -DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) +FieldIndexRemover::remove(uint32_t docId, IFieldIndexRemoveListener &listener) { Iterator itr = _store.get(docId); if (itr.valid()) { @@ -34,14 +34,13 @@ DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) } void -DocumentRemover::insert(datastore::EntryRef wordRef, uint32_t docId) +FieldIndexRemover::insert(datastore::EntryRef wordRef, uint32_t docId) { _wordFieldDocTuples.emplace_back(wordRef, docId); } - void -DocumentRemover::flush() +FieldIndexRemover::flush() { if (_wordFieldDocTuples.empty()) { return; @@ -60,5 +59,4 @@ DocumentRemover::flush() _wordFieldDocTuples.clear(); } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h index 5d44a666ff9..19b3353a27a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h @@ -1,22 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "compact_document_words_store.h" -#include "i_document_insert_listener.h" +#include "compact_words_store.h" +#include "i_field_index_insert_listener.h" namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class WordStore; /** - * Class used to remove documents from the memory index dictionary. + * Class used to handle removal of documents from a FieldIndex. + * + * It tracks all {word, docId} tuples that are inserted into the index, + * and when removing a document, all these {word, docId} tuples are sent to the component + * that is doing the actual removal (IFieldIndexRemoveListener). */ -class DocumentRemover : public IDocumentInsertListener -{ +class FieldIndexRemover : public IFieldIndexInsertListener { private: - struct WordFieldDocTuple - { + struct WordFieldDocTuple { datastore::EntryRef _wordRef; uint32_t _docId; WordFieldDocTuple() : @@ -38,22 +40,21 @@ private: return wft._docId; } }; - }; - CompactDocumentWordsStore _store; - CompactDocumentWordsStore::Builder::UP _builder; + CompactWordsStore _store; + CompactWordsStore::Builder::UP _builder; std::vector<WordFieldDocTuple> _wordFieldDocTuples; const WordStore &_wordStore; public: - DocumentRemover(const WordStore &wordStore); - ~DocumentRemover(); - void remove(uint32_t docId, IDocumentRemoveListener &inverter); - CompactDocumentWordsStore &getStore() { return _store; } - const CompactDocumentWordsStore &getStore() const { return _store; } + FieldIndexRemover(const WordStore &wordStore); + ~FieldIndexRemover(); + void remove(uint32_t docId, IFieldIndexRemoveListener &inverter); + CompactWordsStore &getStore() { return _store; } + const CompactWordsStore &getStore() const { return _store; } - // Implements IDocumentInsertListener + // Implements IFieldIndexInsertListener void insert(datastore::EntryRef wordRef, uint32_t docId) override; void flush() override; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index fa261a4e90a..d19f05a98ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -1,25 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include <vespa/document/annotation/alternatespanlist.h> +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/annotation/spantreevisitor.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/bitcompression/compression.h> +#include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/alternatespanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/document/annotation/spantreevisitor.h> +#include <stdexcept> namespace search::memoryindex { @@ -48,23 +48,17 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; -namespace documentinverterkludge { - -namespace linguistics { +namespace documentinverterkludge::linguistics { const vespalib::string SPANTREE_NAME("linguistics"); } -} - using namespace documentinverterkludge; -namespace -{ +namespace { -class SpanFinder : public SpanTreeVisitor -{ +class SpanFinder : public SpanTreeVisitor { public: int32_t begin_pos; int32_t end_pos; @@ -165,7 +159,6 @@ FieldInverter::processAnnotations(const StringFieldValue &value) } } - void FieldInverter::reset() { @@ -228,14 +221,12 @@ FieldInverter::sortWords() } } - void FieldInverter::startElement(int32_t weight) { _elems.push_back(ElemInfo(weight)); // Fill in length later } - void FieldInverter::endElement() { @@ -270,7 +261,6 @@ FieldInverter::saveWord(const vespalib::stringref word) return wordRef; } - uint32_t FieldInverter::saveWord(const document::FieldValue &fv) { @@ -280,7 +270,6 @@ FieldInverter::saveWord(const document::FieldValue &fv) return saveWord(vespalib::stringref(sRef.first, sRef.second)); } - void FieldInverter::remove(const vespalib::stringref word, uint32_t docId) { @@ -289,7 +278,6 @@ FieldInverter::remove(const vespalib::stringref word, uint32_t docId) _positions.emplace_back(wordRef, docId); } - void FieldInverter::processNormalDocTextField(const StringFieldValue &field) { @@ -298,7 +286,6 @@ FieldInverter::processNormalDocTextField(const StringFieldValue &field) endElement(); } - void FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) { @@ -314,7 +301,6 @@ FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) } } - void FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue &field) { @@ -331,7 +317,6 @@ FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue } } - FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) : _fieldId(fieldId), _elem(0u), @@ -352,7 +337,6 @@ FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) { } - void FieldInverter::abortPendingDoc(uint32_t docId) { @@ -365,7 +349,6 @@ FieldInverter::abortPendingDoc(uint32_t docId) } } - void FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, @@ -390,7 +373,6 @@ FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, dstIdx += size; } - void FieldInverter::trimAbortedDocs() { @@ -413,7 +395,6 @@ FieldInverter::trimAbortedDocs() _abortedDocs.clear(); } - void FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) { @@ -424,7 +405,6 @@ FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) endDoc(); } - void FieldInverter::invertNormalDocTextField(const FieldValue &val) { @@ -467,7 +447,6 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) } } - namespace { struct FullRadix { @@ -479,9 +458,8 @@ struct FullRadix { } - void -FieldInverter::applyRemoves(DocumentRemover &remover) +FieldInverter::applyRemoves(FieldIndexRemover &remover) { for (auto docId : _removeDocs) { remover.remove(docId, *this); @@ -489,9 +467,8 @@ FieldInverter::applyRemoves(DocumentRemover &remover) _removeDocs.clear(); } - void -FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) +FieldInverter::pushDocuments(IOrderedFieldIndexInserter &inserter) { trimAbortedDocs(); @@ -568,6 +545,5 @@ FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) reset(); } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 69cfd370041..ecf2f8d8979 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -2,27 +2,25 @@ #pragma once -#include <map> -#include <set> -#include <vespa/document/fieldvalue/document.h> +#include "i_field_index_remove_listener.h" +#include <vespa/document/annotation/span.h> #include <vespa/document/datatype/datatypes.h> -#include <limits> -#include "i_document_remove_listener.h" -#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/span.h> +#include <vespa/searchlib/index/docidandfeatures.h> +#include <limits> +#include <map> +#include <set> namespace search::memoryindex { -class IOrderedDocumentInserter; -class DocumentRemover; +class IOrderedFieldIndexInserter; +class FieldIndexRemover; -class FieldInverter : public IDocumentRemoveListener -{ +class FieldInverter : public IFieldIndexRemoveListener { public: - class PosInfo - { + class PosInfo { public: uint32_t _wordNum; // XXX: Initially word reference uint32_t _docId; @@ -54,7 +52,6 @@ public: { } - PosInfo(uint32_t wordRef, uint32_t docId) : _wordNum(wordRef), @@ -65,22 +62,19 @@ public: { } - bool - removed() const - { - return _elemId == _elemRemoved; - } + bool removed() const { return _elemId == _elemRemoved; } - bool - operator<(const PosInfo &rhs) const - { - if (_wordNum != rhs._wordNum) + bool operator<(const PosInfo &rhs) const { + if (_wordNum != rhs._wordNum) { return _wordNum < rhs._wordNum; - if (_docId != rhs._docId) + } + if (_docId != rhs._docId) { return _docId < rhs._docId; + } if (_elemId != rhs._elemId) { - if (removed() != rhs.removed()) + if (removed() != rhs.removed()) { return removed() && !rhs.removed(); + } return _elemId < rhs._elemId; } return _wordPos < rhs._wordPos; @@ -95,8 +89,7 @@ private: using WordBuffer = vespalib::Array<char>; - class ElemInfo - { + class ElemInfo { public: int32_t _weight; uint32_t _len; @@ -107,18 +100,13 @@ private: { } - void - setLen(uint32_t len) - { - _len = len; - } + void setLen(uint32_t len) { _len = len; } }; using ElemInfoVec = std::vector<ElemInfo>; using PosInfoVec = std::vector<PosInfo>; - class CompareWordRef - { + class CompareWordRef { const char *const _wordBuffer; public: @@ -127,15 +115,11 @@ private: { } - const char * - getWord(uint32_t wordRef) const - { + const char *getWord(uint32_t wordRef) const { return &_wordBuffer[static_cast<size_t>(wordRef) << 2]; } - bool - operator()(const uint32_t lhs, const uint32_t rhs) const - { + bool operator()(const uint32_t lhs, const uint32_t rhs) const { return strcmp(getWord(lhs), getWord(rhs)) < 0; } }; @@ -143,8 +127,7 @@ private: /* * Range in _positions vector used to represent a document put. */ - class PositionRange - { + class PositionRange { uint32_t _start; uint32_t _len; @@ -155,9 +138,7 @@ private: { } - bool - operator<(const PositionRange &rhs) const - { + bool operator<(const PositionRange &rhs) const { if (_start != rhs._start) { return _start < rhs._start; } @@ -202,14 +183,12 @@ public: * * @param weight element weight */ - void - startElement(int32_t weight); + void startElement(int32_t weight); /** * End an element. */ - void - endElement(); + void endElement(); private: /** @@ -220,8 +199,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const vespalib::stringref word); + VESPA_DLL_LOCAL uint32_t saveWord(const vespalib::stringref word); /** * Save field value as word in word buffer. @@ -230,8 +208,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const document::FieldValue &fv); + VESPA_DLL_LOCAL uint32_t saveWord(const document::FieldValue &fv); /** * Get pointer to saved word from a word reference. @@ -240,9 +217,7 @@ private: * * @return saved word */ - const char * - getWordFromRef(uint32_t wordRef) const - { + const char *getWordFromRef(uint32_t wordRef) const { return &_words[static_cast<size_t>(wordRef) << 2]; } @@ -253,9 +228,7 @@ private: * * @return saved word */ - const char * - getWordFromNum(uint32_t wordNum) const - { + const char *getWordFromNum(uint32_t wordNum) const { return getWordFromRef(_wordRefs[wordNum]); } @@ -266,9 +239,7 @@ private: * * @return word number */ - uint32_t - getWordNum(uint32_t wordRef) const - { + uint32_t getWordNum(uint32_t wordRef) const { const char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; return *reinterpret_cast<const uint32_t *>(p); } @@ -279,9 +250,7 @@ private: * @param wordRef word reference * @param wordNum word number */ - void - updateWordNum(uint32_t wordRef, uint32_t wordNum) - { + void updateWordNum(uint32_t wordRef, uint32_t wordNum) { char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; *reinterpret_cast<uint32_t *>(p) = wordNum; } @@ -292,17 +261,12 @@ private: * * @param wordRef word reference */ - void - add(uint32_t wordRef) { + void add(uint32_t wordRef) { _positions.emplace_back(wordRef, _docId, _elem, _wpos, _elems.size() - 1); } - void - stepWordPos() - { - ++_wpos; - } + void stepWordPos() { ++_wpos; } public: VESPA_DLL_LOCAL void @@ -323,30 +287,22 @@ private: * * @return schema used by this index */ - const index::Schema & - getSchema() const - { - return _schema; - } + const index::Schema &getSchema() const { return _schema; } /** * Clear internal memory structures. */ - void - reset(); + void reset(); /** * Calculate word numbers and replace word references with word * numbers in internal memory structures. */ - void - sortWords(); + void sortWords(); - void - moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); + void moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); - void - trimAbortedDocs(); + void trimAbortedDocs(); /* * Abort a pending document that has already been inverted. @@ -354,8 +310,7 @@ private: * @param docId local id for document * */ - void - abortPendingDoc(uint32_t docId); + void abortPendingDoc(uint32_t docId); public: /** @@ -371,42 +326,31 @@ public: * * @param remover document remover */ - void - applyRemoves(DocumentRemover &remover); + void applyRemoves(FieldIndexRemover &remover); /** - * Push inverted documents to memory index structure. + * Push inverted documents to field index structure using the given inserter. * - * Temporary restriction: Currently only one document at a time is - * supported. - * - * @param inserter ordered document inserter + * Temporary restriction: Currently only one document at a time is supported. */ - void - pushDocuments(IOrderedDocumentInserter &inserter); + void pushDocuments(IOrderedFieldIndexInserter &inserter); /* * Invert a normal text field, based on annotations. */ - void - invertField(uint32_t docId, const document::FieldValue::UP &val); + void invertField(uint32_t docId, const document::FieldValue::UP &val); /* * Setup remove of word in old version of document. */ - virtual void - remove(const vespalib::stringref word, uint32_t docId) override; + virtual void remove(const vespalib::stringref word, uint32_t docId) override; - void - removeDocument(uint32_t docId) - { + void removeDocument(uint32_t docId) { abortPendingDoc(docId); _removeDocs.push_back(docId); } - void - startDoc(uint32_t docId) - { + void startDoc(uint32_t docId) { assert(_docId == 0); assert(docId != 0); abortPendingDoc(docId); @@ -416,9 +360,7 @@ public: _wpos = 0; } - void - endDoc() - { + void endDoc() { uint32_t newPosSize = static_cast<uint32_t>(_positions.size()); _pendingDocs.insert({ _docId, { _oldPosSize, newPosSize - _oldPosSize } }); @@ -426,9 +368,7 @@ public: _oldPosSize = newPosSize; } - void - addWord(const vespalib::stringref word) - { + void addWord(const vespalib::stringref word) { uint32_t wordRef = saveWord(word); if (wordRef != 0u) { add(wordRef); diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h deleted file mode 100644 index 194a98ef8ba..00000000000 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once -#include <vespa/searchlib/datastore/entryref.h> - -namespace search::memoryindex { - -/** - * Interface used to track which {wordRef, fieldId} pairs that are - * inserted into the memory index dictionary for a document. - */ -class IDocumentInsertListener -{ -public: - virtual ~IDocumentInsertListener() {} - virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; - virtual void flush() = 0; -}; - -} - diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h new file mode 100644 index 00000000000..0aacfa53c34 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/searchlib/datastore/entryref.h> + +namespace search::memoryindex { + +/** + * Interface used to track which {wordRef, docId} pairs that are inserted into a FieldIndex. + */ +class IFieldIndexInsertListener { +public: + virtual ~IFieldIndexInsertListener() {} + + /** + * Called when a {wordRef, docId} tuple is inserted into the field index. + */ + virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; + + /** + * Called to process the set of {wordRef, docId} tuples inserted since last flush(). + */ + virtual void flush() = 0; +}; + +} + diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h index 436ee0a49e3..4419303a654 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h @@ -7,13 +7,15 @@ namespace search::memoryindex { /** - * Interface used to track which {wordRef, fieldId} pairs that are - * removed from the memory index dictionary for a document. + * Interface used to track which {word, docId} pairs that are removed from a FieldIndex. */ -class IDocumentRemoveListener -{ +class IFieldIndexRemoveListener { public: - virtual ~IDocumentRemoveListener() {} + virtual ~IFieldIndexRemoveListener() {} + + /** + * Called when a {word, docId} tuple is removed from the field index. + */ virtual void remove(const vespalib::stringref word, uint32_t docId) = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h index 9edd1eb4d3b..a1eee2e10ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h @@ -10,13 +10,14 @@ namespace search::index { class DocIdAndFeatures; } namespace search::memoryindex { /** - * Interface class for ordered document inserter. + * Interface used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class IOrderedDocumentInserter { +class IOrderedFieldIndexInserter { public: - virtual ~IOrderedDocumentInserter() {} + virtual ~IOrderedFieldIndexInserter() {} /** * Set next word to operate on. @@ -24,7 +25,7 @@ public: virtual void setNextWord(const vespalib::stringref word) = 0; /** - * Add (word, docId) tuple with given features. + * Add (word, docId) tuple with the given features. */ virtual void add(uint32_t docId, const index::DocIdAndFeatures &features) = 0; @@ -33,15 +34,13 @@ public: */ virtual void remove(uint32_t docId) = 0; - /* - * Flush pending changes to postinglist for (_word). - * - * _dItr is located at correct position. + /** + * Flush pending changes for the current word (into the underlying posting list). */ virtual void flush() = 0; - /* - * Rewind iterator, to start new pass. + /** + * Rewind to prepare for another set of (word, docId) tuples. */ virtual void rewind() = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 90036c83efb..3ff2d553a96 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" +#include "document_inverter.h" #include "field_index_collection.h" -#include "memoryindex.h" -#include "postingiterator.h" +#include "memory_index.h" +#include "posting_iterator.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -15,7 +15,7 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.memoryindex"); +LOG_SETUP(".searchlib.memoryindex.memory_index"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; @@ -118,7 +118,6 @@ MemoryIndex::commit(const std::shared_ptr<IDestructorCallback> &onWriteDone) flipInverter(); } - void MemoryIndex::flipInverter() { @@ -139,8 +138,7 @@ MemoryIndex::dump(IndexBuilder &indexBuilder) namespace { -class MemTermBlueprint : public queryeval::SimpleLeafBlueprint -{ +class MemTermBlueprint : public queryeval::SimpleLeafBlueprint { private: GenerationHandler::Guard _genGuard; FieldIndex::PostingList::ConstIterator _pitr; @@ -167,8 +165,7 @@ public: setEstimate(estimate); } - SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { auto search = std::make_unique<PostingIterator>(_pitr, _featureStore, _fieldId, tfmda); if (_useBitVector) { LOG(debug, "Return BooleanMatchIteratorWrapper: fieldId(%u), docCount(%zu)", @@ -185,8 +182,7 @@ public: /** * Determines the correct Blueprint to use. **/ -class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper -{ +class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper { private: const FieldSpec &_field; const uint32_t _fieldId; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 621c72d56a3..0b74e05c619 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -23,8 +23,7 @@ class FieldIndexCollection; * Lock-free implementation of a memory-based index * using the document inverter and dictionary classes from searchlib. **/ -class MemoryIndex : public queryeval::Searchable -{ +class MemoryIndex : public queryeval::Searchable { private: index::Schema _schema; ISequencedTaskExecutor &_invertThreads; @@ -136,15 +135,13 @@ public: void dump(index::IndexBuilder &indexBuilder); // implements Searchable - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpec &field, - const query::Node &term) override; - - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpecList &fields, - const query::Node &term) override { + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpec &field, + const query::Node &term) override; + + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpecList &fields, + const query::Node &term) override { return queryeval::Searchable::createBlueprint(requestContext, fields, term); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp index 3c4fca5b044..9b127a8b096 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "ordereddocumentinserter.h" -#include "i_document_insert_listener.h" +#include "i_field_index_insert_listener.h" +#include "ordered_field_index_inserter.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -27,7 +27,7 @@ const vespalib::string emptyWord = ""; } -OrderedDocumentInserter::OrderedDocumentInserter(FieldIndex &fieldIndex) +OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex) : _word(), _prevDocId(noDocId), _prevAdd(false), @@ -39,14 +39,13 @@ OrderedDocumentInserter::OrderedDocumentInserter(FieldIndex &fieldIndex) { } -OrderedDocumentInserter::~OrderedDocumentInserter() +OrderedFieldIndexInserter::~OrderedFieldIndexInserter() { flush(); } - void -OrderedDocumentInserter::flushWord() +OrderedFieldIndexInserter::flushWord() { if (_removes.empty() && _adds.empty()) { return; @@ -68,17 +67,15 @@ OrderedDocumentInserter::flushWord() _adds.clear(); } - void -OrderedDocumentInserter::flush() +OrderedFieldIndexInserter::flush() { flushWord(); _listener.flush(); } - void -OrderedDocumentInserter::setNextWord(const vespalib::stringref word) +OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) { // TODO: Adjust here if zero length words should be legal. assert(_word < word); @@ -103,10 +100,9 @@ OrderedDocumentInserter::setNextWord(const vespalib::stringref word) assert(_word == wordStore.getWord(_dItr.getKey()._wordRef)); } - void -OrderedDocumentInserter::add(uint32_t docId, - const index::DocIdAndFeatures &features) +OrderedFieldIndexInserter::add(uint32_t docId, + const index::DocIdAndFeatures &features) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId || @@ -118,9 +114,8 @@ OrderedDocumentInserter::add(uint32_t docId, _prevAdd = true; } - void -OrderedDocumentInserter::remove(uint32_t docId) +OrderedFieldIndexInserter::remove(uint32_t docId) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId); @@ -129,9 +124,8 @@ OrderedDocumentInserter::remove(uint32_t docId) _prevAdd = false; } - void -OrderedDocumentInserter::rewind() +OrderedFieldIndexInserter::rewind() { assert(_removes.empty() && _adds.empty()); _word = ""; @@ -140,9 +134,8 @@ OrderedDocumentInserter::rewind() _dItr.begin(); } - datastore::EntryRef -OrderedDocumentInserter::getWordRef() const +OrderedFieldIndexInserter::getWordRef() const { return _dItr.getKey()._wordRef; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h index 328346e9eee..03cf3723f01 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h @@ -2,23 +2,25 @@ #pragma once -#include "iordereddocumentinserter.h" +#include "i_ordered_field_index_inserter.h" #include "field_index.h" #include <limits> namespace search::memoryindex { -class IDocumentInsertListener; - +class IFieldIndexInsertListener; /** - * Class for inserting updates to FieldIndex in an ordered manner - * (single pass scan of dictionary tree) + * Class used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. + * + * This is done by doing a single pass scan of the dictionary of the FieldIndex, + * and for each word updating the posting list with docId adds / removes. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { +private: vespalib::stringref _word; uint32_t _prevDocId; bool _prevAdd; @@ -29,7 +31,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter using PostingListKeyDataType = FieldIndex::PostingListKeyDataType; FieldIndex &_fieldIndex; DictionaryTree::Iterator _dItr; - IDocumentInsertListener &_listener; + IFieldIndexInsertListener &_listener; // Pending changes to posting list for (_word) std::vector<uint32_t> _removes; @@ -39,7 +41,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter static constexpr uint32_t noFieldId = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t noDocId = std::numeric_limits<uint32_t>::max(); - /* + /** * Flush pending changes to postinglist for (_word). * * _dItr is located at correct position. @@ -47,13 +49,13 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter void flushWord(); public: - OrderedDocumentInserter(FieldIndex &fieldIndex); - ~OrderedDocumentInserter() override; + OrderedFieldIndexInserter(FieldIndex &fieldIndex); + ~OrderedFieldIndexInserter() override; void setNextWord(const vespalib::stringref word) override; void add(uint32_t docId, const index::DocIdAndFeatures &features) override; void remove(uint32_t docId) override; - /* + /** * Flush pending changes to postinglist for (_word). Also flush * insert listener. * @@ -61,7 +63,7 @@ public: */ void flush() override; - /* + /** * Rewind iterator, to start new pass. */ void rewind() override; diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index ca56299f906..4c29ec321e3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "postingiterator.h" +#include "posting_iterator.h" +#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.postingiterator"); +LOG_SETUP(".searchlib.memoryindex.posting_iterator"); namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h index 2838c65c5eb..de337ef49f3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h @@ -10,8 +10,7 @@ namespace search::memoryindex { /** * Search iterator for memory field index posting list. */ -class PostingIterator : public queryeval::RankedSearchIteratorBase -{ +class PostingIterator : public queryeval::RankedSearchIteratorBase { private: FieldIndex::PostingList::ConstIterator _itr; const FeatureStore &_featureStore; diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index 2c290f17782..c185ec93c9d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -1,20 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "urlfieldinverter.h" -#include "fieldinverter.h" +#include "field_inverter.h" +#include "url_field_inverter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> +#include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.urlfieldinverter"); +LOG_SETUP(".memoryindex.url_field_inverter"); namespace search::memoryindex { @@ -46,7 +46,6 @@ lowercaseToken(vespalib::string &dest, const char *src, size_t srcSize) } - using document::ArrayFieldValue; using document::DataType; using document::FieldValue; @@ -61,7 +60,6 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; - void UrlFieldInverter::startDoc(uint32_t docId) { @@ -75,7 +73,6 @@ UrlFieldInverter::startDoc(uint32_t docId) _hostname->startDoc(docId); } - void UrlFieldInverter::endDoc() { @@ -89,7 +86,6 @@ UrlFieldInverter::endDoc() _hostname->endDoc(); } - void UrlFieldInverter::startElement(int32_t weight) { @@ -103,7 +99,6 @@ UrlFieldInverter::startElement(int32_t weight) _hostname->startElement(weight); } - void UrlFieldInverter::endElement() { @@ -117,7 +112,6 @@ UrlFieldInverter::endElement() _hostname->endElement(); } - void UrlFieldInverter::processUrlSubField(FieldInverter *inverter, const StructFieldValue &field, @@ -145,7 +139,6 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, } } - void UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) { @@ -159,7 +152,6 @@ UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) processUrlSubField(_hostname, field, UrlDataType::FIELD_HOST, true); } - void UrlFieldInverter::processUrlField(const FieldValue &url_field) { @@ -207,7 +199,9 @@ UrlFieldInverter::processUrlField(const FieldValue &url_field) processUrlOldStyle(s); } -void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { +void +UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) +{ URL url(reinterpret_cast<const unsigned char *>(s.data()), s.size()); _hostname->addWord(HOSTNAME_BEGIN); @@ -264,7 +258,6 @@ void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { _hostname->addWord(HOSTNAME_END); } - void UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) { @@ -276,7 +269,6 @@ UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) } } - void UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) { @@ -292,13 +284,16 @@ UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) } namespace { -bool isUriType(const DataType &type) { + +bool +isUriType(const DataType &type) +{ return type == UrlDataType::getInstance() - || type == *DataType::STRING - || type == *DataType::URI; + || type == *DataType::STRING + || type == *DataType::URI; } -} // namespace +} void UrlFieldInverter::invertUrlField(const FieldValue &val) @@ -366,7 +361,6 @@ UrlFieldInverter::removeDocument(uint32_t docId) _hostname->removeDocument(docId); } - UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, FieldInverter *all, FieldInverter *scheme, @@ -389,6 +383,5 @@ UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, { } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h index c902feaf5a6..1659e460af3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h @@ -9,8 +9,7 @@ namespace search::memoryindex { class FieldInverter; -class UrlFieldInverter -{ +class UrlFieldInverter { FieldInverter *_all; FieldInverter *_scheme; FieldInverter *_host; @@ -31,11 +30,10 @@ class UrlFieldInverter void endElement(); - void - processUrlSubField(FieldInverter *inverter, - const document::StructFieldValue &field, - vespalib::stringref subField, - bool addAnchors); + void processUrlSubField(FieldInverter *inverter, + const document::StructFieldValue &field, + vespalib::stringref subField, + bool addAnchors); void processAnnotatedUrlField(const document::StructFieldValue &field); diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index b65fe192e58..ffdc26f5eb0 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wordstore.h" +#include "word_store.h" #include <vespa/searchlib/datastore/datastore.hpp> namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h b/searchlib/src/vespa/searchlib/memoryindex/word_store.h index b909f26157f..4c1526df527 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.h @@ -7,8 +7,7 @@ namespace search::memoryindex { -class WordStore -{ +class WordStore { public: using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; using RefType = DataStoreType::RefType; @@ -23,8 +22,7 @@ public: WordStore(); ~WordStore(); datastore::EntryRef addWord(const vespalib::stringref word); - const char * getWord(datastore::EntryRef ref) const - { + const char *getWord(datastore::EntryRef ref) const { RefType internalRef(ref); return _store.getEntry<char>(internalRef); } diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.h b/searchlib/src/vespa/searchlib/predicate/document_features_store.h index 2cf3e15337a..4c55b67cb11 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.h +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.h @@ -4,7 +4,7 @@ #include "predicate_tree_annotator.h" #include <vespa/searchlib/btree/btree.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/hash_map.h> #include <unordered_set> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 2d7a9abbf79..9cbbd136148 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -2,13 +2,13 @@ #include "fakememtreeocc.h" #include "fpfactory.h" -#include <vespa/searchlib/queryeval/iterators.h> -#include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> +#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/queryeval/iterators.h> #include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index 7fa46fc7531..f0363500559 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -4,7 +4,7 @@ #include "fakeword.h" #include "fakeposting.h" #include "fpfactory.h" -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> #include <vespa/searchlib/memoryindex/field_index.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h index 4802a7571c2..08473f9fc6c 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h @@ -2,20 +2,12 @@ #pragma once -#include <vespa/searchlib/memoryindex/iordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h> #include <sstream> -namespace search -{ +namespace search::memoryindex::test { -namespace memoryindex -{ - -namespace test -{ - -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { std::stringstream _ss; bool _first; bool _verbose; @@ -31,7 +23,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter } } public: - OrderedDocumentInserter() + OrderedFieldIndexInserter() : _ss(), _first(true), _verbose(false), @@ -115,6 +107,4 @@ public: void setVerbose() { _verbose = true; } }; -} // namespace test -} // namespace memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 26c66f43993..155bb194f97 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -409,88 +409,6 @@ ComprFileReadContext::referenceWriteContext(const ComprFileWriteContext &rhs) } -void -ComprFileReadContext::copyWriteContext(const ComprFileWriteContext &rhs) -{ - ComprFileEncodeContext *e = rhs.getEncodeContext(); - ComprFileDecodeContext *d = getDecodeContext(); - - assert(e != NULL); - int usedUnits = e->getUsedUnits(rhs._comprBuf); - assert(usedUnits >= 0); - - dropComprBuf(); - allocComprBuf(usedUnits, 32768); - assert(_comprBufSize >= static_cast<unsigned int>(usedUnits)); - memcpy(_comprBuf, rhs._comprBuf, - static_cast<size_t>(usedUnits) * _unitSize); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - - -void -ComprFileReadContext::referenceReadContext(const ComprFileReadContext &rhs) -{ - ComprFileDecodeContext *d = getDecodeContext(); - - int usedUnits = rhs.getBufferEndFilePos() / _unitSize; - assert(usedUnits >= 0); - assert(static_cast<uint64_t>(usedUnits) * _unitSize == - rhs.getBufferEndFilePos()); - - referenceComprBuf(rhs); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - - -void -ComprFileReadContext::copyReadContext(const ComprFileReadContext &rhs) -{ - ComprFileDecodeContext *d = getDecodeContext(); - - int usedUnits = rhs.getBufferEndFilePos() / _unitSize; - assert(usedUnits >= 0); - assert(static_cast<uint64_t>(usedUnits) * _unitSize == - rhs.getBufferEndFilePos()); - - dropComprBuf(); - allocComprBuf(usedUnits, 32768); - assert(_comprBufSize >= static_cast<unsigned int>(usedUnits)); - memcpy(_comprBuf, rhs._comprBuf, - static_cast<size_t>(usedUnits) * _unitSize); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - ComprFileWriteContext:: ComprFileWriteContext(ComprFileEncodeContext &encodeContext) : ComprBuffer(encodeContext.getUnitByteSize()), diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index 3d44f088c74..d4de1d305fa 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -136,23 +136,7 @@ public: * For unit testing only. Reference data owned by rhs, only works as * long as rhs is live and unchanged. */ - void referenceReadContext(const ComprFileReadContext &rhs); - - /* - * For unit testing only. Copy data owned by rhs. - */ - void copyReadContext(const ComprFileReadContext &rhs); - - /* - * For unit testing only. Reference data owned by rhs, only works as - * long as rhs is live and unchanged. - */ void referenceWriteContext(const ComprFileWriteContext &rhs); - - /* - * For unit testing only. Copy data owned by rhs. - */ - void copyWriteContext(const ComprFileWriteContext &rhs); }; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index d81c9f064b1..da3bd18440b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; @@ -55,7 +55,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); HttpUriRequest request = RequestBuilder.put() .setUri(uri) @@ -66,7 +66,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); HttpUriRequest request = RequestBuilder.delete() .setUri(uri) @@ -76,7 +76,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); HttpUriRequest request = RequestBuilder.put() .setUri(uri) @@ -87,7 +87,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); HttpUriRequest request = RequestBuilder.delete() .setUri(uri) diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index cf044edeac0..e78478bc1a2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -17,13 +17,13 @@ import java.util.Set; */ public interface ZmsClient extends AutoCloseable { - void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token); - void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token); - void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); + void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); - void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token); + void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token); boolean getMembership(AthenzRole role, AthenzIdentity identity); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java index dccd18fed61..a67bd4dcad6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java @@ -33,7 +33,7 @@ public class ProviderResourceGroupRolesRequestEntity { @JsonProperty("resourceGroup") private final String resourceGroup; - public ProviderResourceGroupRolesRequestEntity(AthenzService providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { + public ProviderResourceGroupRolesRequestEntity(AthenzIdentity providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { this.domain = providerService.getDomainName(); this.service = providerService.getName(); this.tenant = tenantDomain.getName(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java index 7883a505c71..6e1987130f2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.client.zms.bindings; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import java.util.List; @@ -23,7 +24,7 @@ public class TenancyRequestEntity { @JsonInclude(JsonInclude.Include.NON_EMPTY) private final List<String> resourceGroups; - public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzService providerService, List<String> resourceGroups) { + public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzIdentity providerService, List<String> resourceGroups) { this.tenantDomain = tenantDomain.getName(); this.providerService = providerService.getFullName(); this.resourceGroups = resourceGroups; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 05395947fc1..ddba229d8d1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -1,10 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.NToken; @@ -22,7 +22,6 @@ import com.yahoo.vespa.athenz.client.zts.bindings.RoleTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.TenantDomainsResponseEntity; import com.yahoo.vespa.athenz.client.zts.utils.IdentityCsrGenerator; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.security.Pkcs10Csr; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -65,8 +64,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public InstanceIdentity registerInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceIdentity registerInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, @@ -81,8 +80,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public InstanceIdentity refreshInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { @@ -101,7 +100,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr) { URI uri = ztsUrl.resolve(String.format("instance/%s/%s/refresh", identity.getDomainName(), identity.getName())); HttpUriRequest request = RequestBuilder.post() .setUri(uri) @@ -114,7 +113,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix) { Pkcs10Csr csr = new IdentityCsrGenerator(dnsSuffix).generateIdentityCsr(identity, keyPair); return getServiceIdentity(identity, keyId, csr); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index 7b77fccfed6..efe244d500f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -28,8 +28,8 @@ public interface ZtsClient extends AutoCloseable { * @param attestationData The signed identity documented serialized to a string. * @return A x509 certificate + service token (optional) */ - InstanceIdentity registerInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + InstanceIdentity registerInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, // TODO Remove this parameter (unused/unnecessary) String attestationData, boolean requestServiceToken, @@ -40,8 +40,8 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate + service token (optional) */ - InstanceIdentity refreshInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr); @@ -51,7 +51,7 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate with CA certificates */ - Identity getServiceIdentity(AthenzService identity, + Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr); @@ -60,7 +60,7 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate with CA certificates */ - Identity getServiceIdentity(AthenzService identity, + Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java index 49d9bb1ec5c..67a49059776 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.Pkcs10CsrUtils; @@ -32,8 +33,8 @@ public class InstanceRegisterInformation { @JsonProperty("token") private final boolean token; - public InstanceRegisterInformation(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceRegisterInformation(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String attestationData, Pkcs10Csr csr, boolean requestServiceToken) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java index b2af2d732bf..d1383bd04fd 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.security.Pkcs10Csr; @@ -12,7 +13,7 @@ import java.security.KeyPair; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_RSA; /** - * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzService, String, Pkcs10Csr)} + * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzIdentity, String, Pkcs10Csr)} * * @author bjorncs */ @@ -24,7 +25,7 @@ public class IdentityCsrGenerator { this.dnsSuffix = dnsSuffix; } - public Pkcs10Csr generateIdentityCsr(AthenzService identity, KeyPair keypair) { + public Pkcs10Csr generateIdentityCsr(AthenzIdentity identity, KeyPair keypair) { return Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=" + identity.getFullName()), keypair, SHA256_WITH_RSA) .addSubjectAlternativeName(String.format( "%s.%s.%s", diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java index 6b318fb16be..e5ed885b316 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identity; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import javax.net.ssl.SSLContext; @@ -13,6 +14,6 @@ import javax.net.ssl.SSLContext; * @author bjorncs */ public interface ServiceIdentityProvider { - AthenzService identity(); + AthenzIdentity identity(); SSLContext getIdentitySslContext(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index d8fa910aa73..2b0e50ed982 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identity; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyStoreType; import com.yahoo.security.SslContextBuilder; @@ -33,7 +34,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde private static final Duration REFRESH_INTERVAL = Duration.ofHours(1); private final AtomicReference<SSLContext> sslContext = new AtomicReference<>(); - private final AthenzService service; + private final AthenzIdentity service; private final File privateKeyFile; private final File certificateFile; private final File trustStoreFile; @@ -48,7 +49,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde createScheduler()); } - public SiaIdentityProvider(AthenzService service, + public SiaIdentityProvider(AthenzIdentity service, Path siaPath, File trustStoreFile) { this(service, @@ -58,7 +59,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde createScheduler()); } - public SiaIdentityProvider(AthenzService service, + public SiaIdentityProvider(AthenzIdentity service, File privateKeyFile, File certificateFile, File trustStoreFile, @@ -81,7 +82,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde } @Override - public AthenzService identity() { + public AthenzIdentity identity() { return service; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index cd35a204b00..40f12b9c6db 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; @@ -31,31 +32,31 @@ public class SiaUtils { private SiaUtils() {} - public static Path getPrivateKeyFile(AthenzService service) { + public static Path getPrivateKeyFile(AthenzIdentity service) { return getPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } - public static Path getPrivateKeyFile(Path root, AthenzService service) { + public static Path getPrivateKeyFile(Path root, AthenzIdentity service) { return root .resolve("keys") .resolve(String.format("%s.%s.key.pem", service.getDomainName(), service.getName())); } - public static Path getCertificateFile(AthenzService service) { + public static Path getCertificateFile(AthenzIdentity service) { return getCertificateFile(DEFAULT_SIA_DIRECTORY, service); } - public static Path getCertificateFile(Path root, AthenzService service) { + public static Path getCertificateFile(Path root, AthenzIdentity service) { return root .resolve("certs") .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); } - public static Optional<PrivateKey> readPrivateKeyFile(AthenzService service) { + public static Optional<PrivateKey> readPrivateKeyFile(AthenzIdentity service) { return readPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } - public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzService service) { + public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzIdentity service) { try { Path privateKeyFile = getPrivateKeyFile(root, service); if (Files.notExists(privateKeyFile)) return Optional.empty(); @@ -65,11 +66,11 @@ public class SiaUtils { } } - public static Optional<X509Certificate> readCertificateFile(AthenzService service) { + public static Optional<X509Certificate> readCertificateFile(AthenzIdentity service) { return readCertificateFile(DEFAULT_SIA_DIRECTORY, service); } - public static Optional<X509Certificate> readCertificateFile(Path root, AthenzService service) { + public static Optional<X509Certificate> readCertificateFile(Path root, AthenzIdentity service) { try { Path certificateFile = getCertificateFile(root, service); if (Files.notExists(certificateFile)) return Optional.empty(); @@ -79,11 +80,11 @@ public class SiaUtils { } } - public static void writePrivateKeyFile(AthenzService service, PrivateKey privateKey) { + public static void writePrivateKeyFile(AthenzIdentity service, PrivateKey privateKey) { writePrivateKeyFile(DEFAULT_SIA_DIRECTORY, service, privateKey); } - public static void writePrivateKeyFile(Path root, AthenzService service, PrivateKey privateKey) { + public static void writePrivateKeyFile(Path root, AthenzIdentity service, PrivateKey privateKey) { try { Path privateKeyFile = getPrivateKeyFile(root, service); Files.createDirectories(privateKeyFile.getParent()); @@ -95,11 +96,11 @@ public class SiaUtils { } } - public static void writeCertificateFile(AthenzService service, X509Certificate certificate) { + public static void writeCertificateFile(AthenzIdentity service, X509Certificate certificate) { writeCertificateFile(DEFAULT_SIA_DIRECTORY, service, certificate); } - public static void writeCertificateFile(Path root, AthenzService service, X509Certificate certificate) { + public static void writeCertificateFile(Path root, AthenzIdentity service, X509Certificate certificate) { try { Path certificateFile = getCertificateFile(root, service); Files.createDirectories(certificateFile.getParent()); @@ -111,11 +112,11 @@ public class SiaUtils { } } - public static List<AthenzService> findSiaServices() { + public static List<AthenzIdentity> findSiaServices() { return findSiaServices(DEFAULT_SIA_DIRECTORY); } - public static List<AthenzService> findSiaServices(Path root) { + public static List<AthenzIdentity> findSiaServices(Path root) { String keyFileSuffix = ".key.pem"; Path keysDirectory = root.resolve("keys"); if ( ! Files.exists(keysDirectory)) diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java index f69e937f294..0e6aff1eeca 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import org.junit.Rule; import org.junit.Test; @@ -35,7 +36,7 @@ public class SiaUtilsTest { AthenzService barService = new AthenzService("my.domain.bar"); Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, barService)); - List<AthenzService> siaIdentities = SiaUtils.findSiaServices(siaRoot); + List<AthenzIdentity> siaIdentities = SiaUtils.findSiaServices(siaRoot); assertThat(siaIdentities.size(), equalTo(2)); assertThat(siaIdentities, hasItem(fooService)); assertThat(siaIdentities, hasItem(barService)); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 4eaedb53c5b..b9219d8f267 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -169,7 +169,7 @@ public class CommandLineArguments { @Option(name = {"--numPersistentConnectionsPerEndpoint"}, description = "How many tcp connections to establish per endoint.)") - private int numPersistentConnectionsPerEndpoint = 1; + private int numPersistentConnectionsPerEndpoint = 4; @Option(name = {"--maxChunkSizeBytes"}, description = "How much data to send to gateway in each message.") diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java index 407c95e5822..02509626176 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java @@ -18,7 +18,7 @@ import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.*; +import static org.junit.Assert.assertThat; public class CommandLineArgumentsTest { @@ -91,7 +91,7 @@ public class CommandLineArgumentsTest { assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080)); assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); assertThat(params.getConnectionParams().getUseCompression(), is(false)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(1)); + assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4)); assertThat(params.getFeedParams().getRoute(), is("default")); assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); @@ -106,6 +106,7 @@ public class CommandLineArgumentsTest { add("host", "hostValue"); add("port", "1234"); add("timeout", "2345"); + add("numPersistentConnectionsPerEndpoint", "7"); args.add("--useCompression"); args.add("--useDynamicThrottling"); add("maxpending", "3456"); @@ -125,6 +126,7 @@ public class CommandLineArgumentsTest { assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L)); assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456)); assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L)); + assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7)); } @Test diff --git a/zkfacade/src/main/sh/zkcat b/zkfacade/src/main/sh/zkcat index 675aeabbcfb..ba8924b456f 100755 --- a/zkfacade/src/main/sh/zkcat +++ b/zkfacade/src/main/sh/zkcat @@ -70,4 +70,4 @@ findhost # END environment bootstrap section -$VESPA_HOME/bin/zkctl get $@ +$VESPA_HOME/bin/vespa-zkctl get $@ diff --git a/zkfacade/src/main/sh/zkctl b/zkfacade/src/main/sh/zkctl index 721becf8c79..43112abd925 100755 --- a/zkfacade/src/main/sh/zkctl +++ b/zkfacade/src/main/sh/zkctl @@ -71,4 +71,4 @@ findhost # END environment bootstrap section # Get rid of the interactive zkcli prompt by running it in a subshell -(echo "$@" | $VESPA_HOME/bin/zkcli) +(echo "$@" | $VESPA_HOME/bin/vespa-zkcli) diff --git a/zkfacade/src/main/sh/zkls b/zkfacade/src/main/sh/zkls index c75f63efbde..5ab1204cea8 100755 --- a/zkfacade/src/main/sh/zkls +++ b/zkfacade/src/main/sh/zkls @@ -70,4 +70,4 @@ findhost # END environment bootstrap section -$VESPA_HOME/bin/zkctl ls $@ +$VESPA_HOME/bin/vespa-zkctl ls $@ |