diff options
348 files changed, 4683 insertions, 3962 deletions
diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index 8f6bf953e6f..4352c8b9c0a 100755 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -23,7 +23,7 @@ while getopts "uh" opt; do done shift $((OPTIND-1)) -if [[ $# -eq 0 ]]; then +if [[ $# -eq 0 ]]; then SOURCE_DIR=$(dirname "$0") EXTRA_CMAKE_ARGS="" elif [[ $# -eq 1 ]]; then diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 46acdb44746..d5e42276c1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.clustercontroller; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; @@ -10,10 +11,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.container.Container; -import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; @@ -21,6 +20,8 @@ import com.yahoo.vespa.model.container.component.Handler; import java.util.Set; import java.util.TreeSet; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + /** * Container implementation for cluster-controllers */ @@ -87,8 +88,7 @@ public class ClusterControllerContainer extends Container implements } private void addHandler(Handler h, String binding) { - h.addServerBindings("http://*/" + binding, - "https://*/" + binding); + h.addServerBindings("http://*/" + binding); super.addHandler(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 1ec72c6a2c5..09b0015540f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -101,28 +101,36 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("handled.requests.count")); - metrics.add(new Metric("handled.latency.average")); + metrics.add(new Metric("handled.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("handled.latency.max")); + metrics.add(new Metric("handled.latency.sum")); + metrics.add(new Metric("handled.latency.count")); metrics.add(new Metric("serverRejectedRequests.rate")); metrics.add(new Metric("serverRejectedRequests.count")); - metrics.add(new Metric("serverThreadPoolSize.average")); - metrics.add(new Metric("serverThreadPoolSize.min")); - metrics.add(new Metric("serverThreadPoolSize.max")); - metrics.add(new Metric("serverThreadPoolSize.rate")); - metrics.add(new Metric("serverThreadPoolSize.count")); + metrics.add(new Metric("serverThreadPoolSize.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.min")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.rate")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverThreadPoolSize.last")); - metrics.add(new Metric("serverActiveThreads.average")); + metrics.add(new Metric("serverActiveThreads.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverActiveThreads.min")); metrics.add(new Metric("serverActiveThreads.max")); - metrics.add(new Metric("serverActiveThreads.rate")); + metrics.add(new Metric("serverActiveThreads.rate")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); - metrics.add(new Metric("httpapi_latency.average")); - metrics.add(new Metric("httpapi_pending.average")); + metrics.add(new Metric("httpapi_latency.max")); + metrics.add(new Metric("httpapi_latency.sum")); + metrics.add(new Metric("httpapi_latency.count")); + metrics.add(new Metric("httpapi_latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("httpapi_pending.max")); + metrics.add(new Metric("httpapi_pending.sum")); + metrics.add(new Metric("httpapi_pending.count")); + metrics.add(new Metric("httpapi_pending.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("httpapi_num_operations.rate")); metrics.add(new Metric("httpapi_num_updates.rate")); metrics.add(new Metric("httpapi_num_removes.rate")); @@ -133,6 +141,7 @@ public class VespaMetricSet { metrics.add(new Metric("mem.heap.total.average")); metrics.add(new Metric("mem.heap.free.average")); metrics.add(new Metric("mem.heap.used.average")); + metrics.add(new Metric("mem.heap.used.max")); metrics.add(new Metric("jdisc.memory_mappings.max")); metrics.add(new Metric("jdisc.open_file_descriptors.max")); @@ -158,10 +167,14 @@ public class VespaMetricSet { metrics.add(new Metric("http.status.401.rate")); metrics.add(new Metric("http.status.403.rate")); - metrics.add(new Metric("jdisc.http.request.uri_length.average")); metrics.add(new Metric("jdisc.http.request.uri_length.max")); - metrics.add(new Metric("jdisc.http.request.content_size.average")); + metrics.add(new Metric("jdisc.http.request.uri_length.sum")); + metrics.add(new Metric("jdisc.http.request.uri_length.count")); + metrics.add(new Metric("jdisc.http.request.uri_length.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("jdisc.http.request.content_size.max")); + metrics.add(new Metric("jdisc.http.request.content_size.sum")); + metrics.add(new Metric("jdisc.http.request.content_size.count")); + metrics.add(new Metric("jdisc.http.request.content_size.average")); // TODO: Remove in Vespa 8 return metrics; } @@ -197,36 +210,59 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("peak_qps.max")); - metrics.add(new Metric("search_connections.average")); - metrics.add(new Metric("active_queries.average")); - metrics.add(new Metric("feed.latency.average")); + metrics.add(new Metric("search_connections.max")); + metrics.add(new Metric("search_connections.sum")); + metrics.add(new Metric("search_connections.count")); + metrics.add(new Metric("search_connections.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("active_queries.max")); + metrics.add(new Metric("active_queries.sum")); + metrics.add(new Metric("active_queries.count")); + metrics.add(new Metric("active_queries.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("feed.latency.max")); + metrics.add(new Metric("feed.latency.sum")); + metrics.add(new Metric("feed.latency.count")); + metrics.add(new Metric("feed.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("feed.http-requests.count")); metrics.add(new Metric("feed.http-requests.rate")); metrics.add(new Metric("queries.rate")); - metrics.add(new Metric("query_container_latency.average")); - metrics.add(new Metric("query_latency.average")); + metrics.add(new Metric("query_container_latency.max")); + metrics.add(new Metric("query_container_latency.sum")); + metrics.add(new Metric("query_container_latency.count")); + metrics.add(new Metric("query_container_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("query_latency.max")); + metrics.add(new Metric("query_latency.sum")); + metrics.add(new Metric("query_latency.count")); + metrics.add(new Metric("query_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("query_latency.95percentile")); metrics.add(new Metric("query_latency.99percentile")); metrics.add(new Metric("failed_queries.rate")); metrics.add(new Metric("degraded_queries.rate")); - metrics.add(new Metric("hits_per_query.average")); + metrics.add(new Metric("hits_per_query.max")); + metrics.add(new Metric("hits_per_query.sum")); + metrics.add(new Metric("hits_per_query.count")); + metrics.add(new Metric("hits_per_query.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("documents_covered.count")); metrics.add(new Metric("documents_total.count")); metrics.add(new Metric("dispatch_internal.rate")); metrics.add(new Metric("dispatch_fdispatch.rate")); - metrics.add(new Metric("totalhits_per_query.average")); + metrics.add(new Metric("totalhits_per_query.max")); + metrics.add(new Metric("totalhits_per_query.sum")); + metrics.add(new Metric("totalhits_per_query.count")); + metrics.add(new Metric("totalhits_per_query.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("empty_results.rate")); metrics.add(new Metric("requestsOverQuota.rate")); metrics.add(new Metric("requestsOverQuota.count")); - metrics.add(new Metric("relevance.at_1.average")); + metrics.add(new Metric("relevance.at_1.sum")); metrics.add(new Metric("relevance.at_1.count")); - metrics.add(new Metric("relevance.at_3.average")); + metrics.add(new Metric("relevance.at_1.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("relevance.at_3.sum")); metrics.add(new Metric("relevance.at_3.count")); - metrics.add(new Metric("relevance.at_10.average")); + metrics.add(new Metric("relevance.at_3.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("relevance.at_10.sum")); metrics.add(new Metric("relevance.at_10.count")); + metrics.add(new Metric("relevance.at_10.average")); // TODO: Remove in Vespa 8 // Errors from qrserver metrics.add(new Metric("error.timeout.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index a370e0dc38b..446169c0ed2 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -212,9 +212,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT, - "https://*" + StateHandler.STATE_API_ROOT, - "http://*" + StateHandler.STATE_API_ROOT + "/*", - "https://*" + StateHandler.STATE_API_ROOT + "/*"); + "http://*" + StateHandler.STATE_API_ROOT + "/*"); addComponent(stateHandler); } @@ -242,13 +240,13 @@ public abstract class ContainerCluster<CONTAINER extends Container> Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( APPLICATION_STATUS_HANDLER_CLASS, null), null)); - statusHandler.addServerBindings("http://*/ApplicationStatus", "https://*/ApplicationStatus"); + statusHandler.addServerBindings("http://*/ApplicationStatus"); addComponent(statusHandler); } public void addVipHandler() { Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); - vipHandler.addServerBindings("http://*/status.html", "https://*/status.html"); + vipHandler.addServerBindings("http://*/status.html"); addComponent(vipHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index b7c8460a9a5..6b4f8d486ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -49,9 +49,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra public static Handler<?> getHandler() { Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); handler.addServerBindings("http://*/" + REST_BINDING, - "https://*/" + REST_BINDING, - "http://*/" + REST_BINDING + "/*", - "https://*/" + REST_BINDING + "/*"); + "http://*/" + REST_BINDING + "/*"); return handler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 4220a6571a0..4de50fc2dde 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -4,11 +4,11 @@ package com.yahoo.vespa.model.container.http; import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; -import com.yahoo.vespa.model.container.http.Http.Binding; import java.util.ArrayList; import java.util.Collection; @@ -47,10 +47,12 @@ public final class AccessControl { private final Set<String> excludeBindings = new LinkedHashSet<>(); private Collection<Handler<?>> handlers = Collections.emptyList(); private Collection<Servlet> servlets = Collections.emptyList(); + private final DeployLogger logger; - public Builder(String domain, String applicationId) { + public Builder(String domain, String applicationId, DeployLogger logger) { this.domain = domain; this.applicationId = applicationId; + this.logger = logger; } public Builder readEnabled(boolean readEnabled) { @@ -85,7 +87,7 @@ public final class AccessControl { public AccessControl build() { return new AccessControl(domain, applicationId, writeEnabled, readEnabled, - excludeBindings, vespaDomain, servlets, handlers); + excludeBindings, vespaDomain, servlets, handlers, logger); } } @@ -97,6 +99,7 @@ public final class AccessControl { private final Set<String> excludedBindings; private final Collection<Handler<?>> handlers; private final Collection<Servlet> servlets; + private final DeployLogger logger; private AccessControl(String domain, String applicationId, @@ -105,7 +108,8 @@ public final class AccessControl { Set<String> excludedBindings, Optional<String> vespaDomain, Collection<Servlet> servlets, - Collection<Handler<?>> handlers) { + Collection<Handler<?>> handlers, + DeployLogger logger) { this.domain = domain; this.applicationId = applicationId; this.readEnabled = readEnabled; @@ -114,6 +118,7 @@ public final class AccessControl { this.vespaDomain = vespaDomain; this.handlers = handlers; this.servlets = servlets; + this.logger = logger; } public List<Binding> getBindings() { @@ -125,14 +130,14 @@ public final class AccessControl { return handlers.stream() .filter(this::shouldHandlerBeProtected) .flatMap(handler -> handler.getServerBindings().stream()) - .map(AccessControl::accessControlBinding); + .map(binding -> accessControlBinding(binding, logger)); } private Stream<Binding> getServletBindings() { return servlets.stream() .filter(this::shouldServletBeProtected) .flatMap(AccessControl::servletBindings) - .map(AccessControl::accessControlBinding); + .map(binding -> accessControlBinding(binding, logger)); } private boolean shouldHandlerBeProtected(Handler<?> handler) { @@ -148,11 +153,11 @@ public final class AccessControl { return servletBindings(servlet).noneMatch(excludedBindings::contains); } - private static Binding accessControlBinding(String binding) { - return new Binding(new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), binding); + private static Binding accessControlBinding(String binding, DeployLogger logger) { + return Binding.create(new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), binding, logger); } private static Stream<String> servletBindings(Servlet servlet) { - return Stream.of("http://*/", "https://*/").map(protocol -> protocol + servlet.bindingPath); + return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java new file mode 100644 index 00000000000..09ac5f580ed --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java @@ -0,0 +1,38 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.application.api.DeployLogger; + +import java.util.logging.Level; + +/** + * @author bjorncs + */ +public class Binding { + private final ComponentSpecification filterId; + private final String binding; + + private Binding(ComponentSpecification filterId, String binding) { + this.filterId = filterId; + this.binding = binding; + } + + public static Binding create(ComponentSpecification filterId, String binding, DeployLogger logger) { + if (binding.startsWith("https://")) { + logger.log(Level.WARNING, String.format( + "For binding '%s' on '%s': 'https' bindings are deprecated, " + + "use 'http' instead to bind to both http and https traffic.", + binding, filterId)); + } + return new Binding(filterId, binding); + } + + public ComponentSpecification filterId() { + return filterId; + } + + public String binding() { + return binding; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 9e85a889075..9a67871e683 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.jdisc.http.ServerConfig; @@ -20,16 +19,6 @@ import java.util.Optional; */ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { - public static class Binding { - public final ComponentSpecification filterId; - public final String binding; - - public Binding(ComponentSpecification filterId, String binding) { - this.filterId = filterId; - this.binding = binding; - } - } - private FilterChains filterChains; private JettyHttpServer httpServer; public final List<Binding> bindings; @@ -91,8 +80,8 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl for (final Binding binding : bindings) { builder.filter( new ServerConfig.Filter.Builder() - .id(binding.filterId.stringValue()) - .binding(binding.binding)); + .id(binding.filterId().stringValue()) + .binding(binding.binding())); } } @@ -110,8 +99,8 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl ComponentRegistry<Chain<Filter>> chains = filterChains.allChains(); for (Binding binding: bindings) { - if (filters.getComponent(binding.filterId) == null && chains.getComponent(binding.filterId) == null) - throw new RuntimeException("Can't find filter " + binding.filterId + " for binding " + binding.binding); + if (filters.getComponent(binding.filterId()) == null && chains.getComponent(binding.filterId()) == null) + throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 08268f5085d..8cf430741f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; -import com.yahoo.vespa.model.container.http.Http.Binding; +import com.yahoo.vespa.model.container.http.Binding; import org.w3c.dom.Element; import java.util.ArrayList; @@ -41,7 +41,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> Element filteringElem = XML.getChild(spec, "filtering"); if (filteringElem != null) { filterChains = new FilterChainsBuilder().build(deployState, ancestor, filteringElem); - bindings = readFilterBindings(filteringElem); + bindings = readFilterBindings(filteringElem, deployState.getDeployLogger()); Element accessControlElem = XML.getChild(filteringElem, "access-control"); if (accessControlElem != null) { @@ -65,7 +65,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> String application = XmlHelper.getOptionalChildValue(accessControlElem, "application") .orElse(getDeployedApplicationId(deployState, ancestor).value()); - AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), application); + AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), application, deployState.getDeployLogger()); getContainerCluster(ancestor).ifPresent(cluster -> { builder.setHandlers(cluster.getHandlers()); @@ -106,7 +106,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return Optional.of((ApplicationContainerCluster) currentProducer); } - private List<Binding> readFilterBindings(Element filteringSpec) { + private List<Binding> readFilterBindings(Element filteringSpec, DeployLogger logger) { List<Binding> result = new ArrayList<>(); for (Element child: XML.getChildren(filteringSpec)) { @@ -116,7 +116,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> for (Element bindingSpec: XML.getChildren(child, "binding")) { String binding = XML.getValue(bindingSpec); - result.add(new Binding(chainId, binding)); + result.add(Binding.create(chainId, binding, logger)); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java index 32f0f373a92..4fd79a4f335 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java @@ -4,10 +4,6 @@ package com.yahoo.vespa.model.container.processing; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.component.chain.Chains; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - /** * Root config producer for processing * @@ -15,7 +11,7 @@ import java.util.List; */ public class ProcessingChains extends Chains<ProcessingChain> { - public static final String[] defaultBindings = new String[] {"http://*/processing/*", "https://*/processing/*"}; + public static final String[] defaultBindings = new String[] {"http://*/processing/*"}; public ProcessingChains(AbstractConfigProducer parent, String subId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 6663dc0bbc0..7e80c6be221 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -271,7 +271,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_SETTING)); cluster.addComponent( new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), - "http://*/" + name, "https://*/" + name)); + "http://*/" + name)); } else { cluster.addVipHandler(); } @@ -712,7 +712,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); - String[] defaultBindings = {"http://*/search/*", "https://*/search/*"}; + String[] defaultBindings = {"http://*/search/*"}; for (String binding: serverBindings(searchElement, defaultBindings)) { searchHandler.addServerBindings(binding); } @@ -722,7 +722,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addGUIHandler(ApplicationContainerCluster cluster) { Handler<?> guiHandler = new GUIHandler(); - guiHandler.addServerBindings("http://"+GUIHandler.BINDING, "https://"+GUIHandler.BINDING); + guiHandler.addServerBindings("http://"+GUIHandler.BINDING); cluster.addComponent(guiHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index a44f5440a9f..10e24a609f7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -18,7 +18,7 @@ import java.util.logging.Logger; public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); - private static final String[] DEFAULT_BINDINGS = {"http://*/", "https://*/"}; + private static final String[] DEFAULT_BINDINGS = {"http://*/"}; public static ContainerDocumentApi.Options build(Element spec) { return new ContainerDocumentApi.Options( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java index 4d04a1be614..b6d63af180d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java @@ -52,9 +52,9 @@ public class FilterBindingsTest extends DomBuilderTest { "</http>"); Http http = buildHttp(xml); - Http.Binding binding = first(http.bindings); - assertThat(binding.filterId.getName(), is("my-request-chain")); - assertThat(binding.binding, is(MY_CHAIN_BINDING)); + Binding binding = first(http.bindings); + assertThat(binding.filterId().getName(), is("my-request-chain")); + assertThat(binding.binding(), is(MY_CHAIN_BINDING)); Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-request-chain"); assertNotNull("Missing chain", myChain); @@ -72,9 +72,9 @@ public class FilterBindingsTest extends DomBuilderTest { "</http>"); Http http = buildHttp(xml); - Http.Binding binding = first(http.bindings); - assertThat(binding.filterId.getName(), is("my-response-chain")); - assertThat(binding.binding, is(MY_CHAIN_BINDING)); + Binding binding = first(http.bindings); + assertThat(binding.filterId().getName(), is("my-response-chain")); + assertThat(binding.binding(), is(MY_CHAIN_BINDING)); Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-response-chain"); assertNotNull("Missing chain", myChain); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index fd1ea2ae7ab..5433c7659cc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.state.StateHandler; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.http.AccessControl; import com.yahoo.vespa.model.container.http.Http; -import com.yahoo.vespa.model.container.http.Http.Binding; +import com.yahoo.vespa.model.container.http.Binding; import com.yahoo.vespa.model.container.http.xml.HttpBuilder; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import org.junit.Test; @@ -140,8 +140,8 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { missingRequiredBindings.isEmpty()); FORBIDDEN_HANDLER_BINDINGS.forEach(forbiddenBinding -> http.getBindings().forEach( - binding -> assertFalse("Access control chain was bound to: " + binding.binding, - binding.binding.contains(forbiddenBinding)))); + binding -> assertFalse("Access control chain was bound to: " + binding.binding(), + binding.binding().contains(forbiddenBinding)))); } @Test @@ -196,8 +196,8 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { @Test public void servlet_can_be_excluded_by_excluding_one_of_its_bindings() throws Exception { final String servletPath = "servlet/path"; - final String notExcludedBinding = "https://*/" + servletPath; - final String excludedBinding = "http://*/" + servletPath; + final String notExcludedBinding = "http://*:8081/" + servletPath; + final String excludedBinding = "http://*:8080/" + servletPath; Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", httpWithExcludedBinding(excludedBinding), @@ -217,8 +217,8 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { @Test public void rest_api_can_be_excluded_by_excluding_one_of_its_bindings() throws Exception { final String restApiPath = "api/v0"; - final String notExcludedBinding = "http://*/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; - final String excludedBinding = "https://*/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; + final String notExcludedBinding = "http://*:8081/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; + final String excludedBinding = "http://*:8080/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", httpWithExcludedBinding(excludedBinding), @@ -256,7 +256,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { private boolean containsBinding(Collection<Binding> bindings, String binding) { for (Binding b : bindings) { - if (b.binding.contains(binding)) + if (b.binding().contains(binding)) return true; } return false; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index 428286c4794..5b92934678d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -16,7 +16,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertThat; @@ -64,7 +63,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa Element elem = DomBuilderTest.parse( "<jdisc id='cluster1' version='1.0'>", " <document-api>", - " <binding>https://*/document-api/</binding>", + " <binding>http://*/document-api/</binding>", " <binding>missing-trailing-slash</binding>", " </document-api>", nodesXml, @@ -77,8 +76,8 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa private void verifyCustomBindings(String id, String bindingSuffix) { Handler<?> handler = getHandlers("cluster1").get(id); - assertThat(handler.getServerBindings(), hasItem("https://*/document-api/" + bindingSuffix)); - assertThat(handler.getServerBindings(), hasItem("https://*/document-api/" + bindingSuffix + "/")); + assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix)); + assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix + "/")); assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix)); assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix + "/")); @@ -102,9 +101,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(4)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(2)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 26820672f2c..31077df7f7c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -35,7 +35,6 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.component.HttpFilter; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Test; @@ -266,11 +265,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName()); assertThat(applicationStatusHandler.serverBindings(), - contains("http://*/ApplicationStatus", "https://*/ApplicationStatus")); + contains("http://*/ApplicationStatus")); JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName()); assertThat(fileRequestHandler.serverBindings(), - contains("http://*/status.html", "https://*/status.html")); + contains("http://*/status.html")); } @Test diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 40a2b31b3fc..d5a02949cf3 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -75,7 +75,6 @@ <access-control domain="my.athens-domain" read="true"> <exclude> <binding>http//*/foo/*</binding> - <binding>https://*/foo/*</binding> </exclude> <application>my-app</application> <vespa-domain>vespa.vespa.cd</vespa-domain> @@ -134,7 +133,6 @@ <document-api> <binding>http://*/document-api/</binding> - <binding>https://*/document-api/</binding> <abortondocumenterror>false</abortondocumenterror> <retryenabled>false</retryenabled> <timeout>5.55</timeout> diff --git a/config-model/src/test/schema-test-files/standalone-container.xml b/config-model/src/test/schema-test-files/standalone-container.xml index e36218fa084..0656f293001 100644 --- a/config-model/src/test/schema-test-files/standalone-container.xml +++ b/config-model/src/test/schema-test-files/standalone-container.xml @@ -70,7 +70,6 @@ <document-api> <binding>http://*/document-api/</binding> - <binding>https://*/document-api/</binding> <abortondocumenterror>false</abortondocumenterror> <retryenabled>false</retryenabled> <timeout>5.55</timeout> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 2de11be08f2..db01bb91b3b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -24,7 +24,7 @@ public enum SystemName { Public, /** VaaS */ - vaas; + vaas; // TODO: Remove this and use public everywhere public static SystemName defaultSystem() { return main; @@ -35,7 +35,7 @@ public enum SystemName { case "dev": return dev; case "cd": return cd; case "main": return main; - case "public": return Public; + case "public": case "Public": return Public; case "vaas": return vaas; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java index 96ff468d2d6..d5831efdbaa 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/UpgradePolicy.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.zone; +package com.yahoo.config.provision.zone; import java.util.ArrayList; import java.util.Arrays; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index 9282e612dac..ab44fdcb05e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.zone; +package com.yahoo.config.provision.zone; import com.yahoo.config.provision.CloudName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java index 10cf862fa30..675b770dc29 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneFilterMock.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.zone; +package com.yahoo.config.provision.zone; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index 962aa1ad93e..7eeae10eeb8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.zone; +package com.yahoo.config.provision.zone; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java index 4205f30e995..49c72b9eea8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneList.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java @@ -1,5 +1,5 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.zone; +package com.yahoo.config.provision.zone; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/package-info.java new file mode 100644 index 00000000000..3e4de2d6b23 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.provision.zone; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index accc313dcae..4c98ca64847 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(configserver) +install_configserver_component(configserver) vespa_install_script(src/main/sh/vespa-configserver-remove-state bin) vespa_install_script(src/main/sh/ping-configserver libexec/vespa) @@ -10,16 +10,15 @@ vespa_install_script(src/main/sh/stop-configserver libexec/vespa) install(DIRECTORY src/main/resources/logd DESTINATION conf) install(DIRECTORY src/main/resources/configserver-app DESTINATION conf) -install(CODE "execute_process(COMMAND mkdir -p \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components)") -install(CODE "execute_process(COMMAND mkdir -p \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/config-models)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/athenz-identity-provider-service.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/config-model-fat.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/config-model-fat.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-flags-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver-flags.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/flags-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/flags.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/orchestrator-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/orchestrator.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/service-monitor-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/service-monitor.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/application-model-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/application-model.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/node-repository-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/node-repository.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/zkfacade-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/zkfacade.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/lib/jars/config-models)") +install(DIRECTORY DESTINATION conf/configserver-app/components) +install(DIRECTORY DESTINATION conf/configserver-app/config-models) +install_symlink(lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar conf/configserver-app/components/athenz-identity-provider-service.jar) +install_symlink(lib/jars/config-model-fat.jar conf/configserver-app/components/config-model-fat.jar) +install_symlink(lib/jars/configserver-flags-jar-with-dependencies.jar conf/configserver-app/components/configserver-flags.jar) +install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/configserver-app/components/flags.jar) +install_symlink(lib/jars/orchestrator-jar-with-dependencies.jar conf/configserver-app/components/orchestrator.jar) +install_symlink(lib/jars/service-monitor-jar-with-dependencies.jar conf/configserver-app/components/service-monitor.jar) +install_symlink(lib/jars/application-model-jar-with-dependencies.jar conf/configserver-app/components/application-model.jar) +install_symlink(lib/jars/node-repository-jar-with-dependencies.jar conf/configserver-app/components/node-repository.jar) +install_symlink(lib/jars/zkfacade-jar-with-dependencies.jar conf/configserver-app/components/zkfacade.jar) +install_symlink(conf/configserver-app/components lib/jars/config-models) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index c4b1d0f7c86..6eaefe4c82b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -301,12 +301,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) return false; TenantApplications tenantApplications = tenant.getApplicationRepo(); - if (!tenantApplications.listApplications().contains(applicationId)) return false; + if (!tenantApplications.activeApplications().contains(applicationId)) return false; // Deleting an application is done by deleting the remote session and waiting // until the config server where the deployment happened picks it up and deletes // the local session - long sessionId = tenantApplications.getSessionIdForApplication(applicationId); + long sessionId = tenantApplications.requireActiveSessionOf(applicationId); RemoteSession remoteSession = getRemoteSession(tenant, sessionId); remoteSession.createDeleteTransaction().commit(); @@ -324,7 +324,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server // this is done asynchronously in application maintenance by the node repository) - transaction.add(tenantApplications.deleteApplication(applicationId)); + transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); @@ -425,7 +425,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<ApplicationId> listApplications() { return tenantRepository.getAllTenants().stream() - .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream()) + .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream()) .collect(Collectors.toSet()); } @@ -483,7 +483,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (applicationRepo == null) throw new IllegalArgumentException("Application repo for tenant '" + tenant.getName() + "' not found"); - return applicationRepo.getSessionIdForApplication(applicationId); + return applicationRepo.requireActiveSessionOf(applicationId); } public void validateThatRemoteSessionIsNotActive(Tenant tenant, long sessionId) { @@ -525,6 +525,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + tenant.getApplicationRepo().createApplication(applicationId); LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); SessionFactory sessionFactory = tenant.getSessionFactory(); LocalSession session = sessionFactory.createSession(applicationDirectory, applicationId, timeoutBudget); @@ -565,7 +566,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private List<ApplicationId> activeApplications(TenantName tenantName) { - return tenantRepository.getTenant(tenantName).getApplicationRepo().listApplications(); + return tenantRepository.getTenant(tenantName).getApplicationRepo().activeApplications(); } // ---------------- Misc operations ---------------------------------------------------------------- @@ -616,7 +617,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); TenantApplications applicationRepo = tenant.getApplicationRepo(); try { - long currentActiveSessionId = applicationRepo.getSessionIdForApplication(appId); + long currentActiveSessionId = applicationRepo.requireActiveSessionOf(appId); RemoteSession currentActiveSession = getRemoteSession(tenant, currentActiveSessionId); if (currentActiveSession != null) { currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); @@ -646,7 +647,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<ApplicationId> listApplicationIds(Tenant tenant) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.listApplications(); + return applicationRepo.activeApplications(); } private void cleanupTempDirectory(File tempDir) { @@ -658,13 +659,13 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private LocalSession getExistingSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return getLocalSession(tenant, applicationRepo.getSessionIdForApplication(applicationId)); + return getLocalSession(tenant, applicationRepo.requireActiveSessionOf(applicationId)); } private LocalSession getActiveSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - if (applicationRepo.listApplications().contains(applicationId)) { - return tenant.getLocalSessionRepo().getSession(applicationRepo.getSessionIdForApplication(applicationId)); + if (applicationRepo.activeApplications().contains(applicationId)) { + return tenant.getLocalSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 3705a0ec145..92add4d053b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -17,20 +17,22 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import java.util.List; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * The applications of a tenant, backed by ZooKeeper. * - * Each application is stored as a single node under /config/v2/tenants/<tenant>/applications/<applications>, - * named the same as the application id and containing the id of the session storing the content of the application. + * Each application is stored under /config/v2/tenants/<tenant>/applications/<applications>, + * the root contains the currently active session, if any, and sub-paths /preparing contains the session id + * of whatever session may be activated next, if any, and /lock is used for synchronizing writes to all these paths. * * @author Ulf Lilleengen + * @author jonmv */ public class TenantApplications { @@ -69,16 +71,19 @@ public class TenantApplications { * * @return a list of {@link ApplicationId}s that are active. */ - public List<ApplicationId> listApplications() { + public List<ApplicationId> activeApplications() { return curator.getChildren(applicationsPath).stream() - .flatMap(this::parseApplication) + .filter(this::isValid) + .sorted() + .map(ApplicationId::fromSerializedForm) + .filter(id -> activeSessionOf(id).isPresent()) .collect(Collectors.toUnmodifiableList()); } - // TODO jvenstad: Remove after it has run once everywhere. - private Stream<ApplicationId> parseApplication(String appNode) { + private boolean isValid(String appNode) { // TODO jvenstad: Remove after it has run once everywhere. try { - return Stream.of(ApplicationId.fromSerializedForm(appNode)); + ApplicationId.fromSerializedForm(appNode); + return true; } catch (IllegalArgumentException __) { log.log(LogLevel.INFO, TenantRepository.logPre(tenant) + "Unable to parse application id from '" + appNode + "'; deleting it as it shouldn't be here."); @@ -88,44 +93,50 @@ public class TenantApplications { catch (Exception e) { log.log(LogLevel.WARNING, TenantRepository.logPre(tenant) + "Failed to clean up stray node '" + appNode + "'!", e); } - return Stream.empty(); + return false; } } + /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */ + public OptionalLong activeSessionOf(ApplicationId id) { + String data = curator.getData(applicationPath(id)).map(Utf8::toString) + .orElseThrow(() -> new IllegalArgumentException("Unknown application '" + id + "'.")); + return data.isEmpty() ? OptionalLong.empty() : OptionalLong.of(Long.parseLong(data)); + } + /** * Returns a transaction which writes the given session id as the currently active for the given application. * * @param applicationId An {@link ApplicationId} that represents an active application. * @param sessionId Id of the session containing the application package for this id. */ - public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { - if (listApplications().contains(applicationId)) { - return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); - } else { - return new CuratorTransaction(curator).add(CuratorOperations.create(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); - } + public Transaction createPutTransaction(ApplicationId applicationId, long sessionId) { + return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + } + + /** + * Creates a node for the given application, marking its existence. + */ + public void createApplication(ApplicationId id) { + curator.create(applicationPath(id)); } /** - * Return the stored session id for a given application. + * Return the active session id for a given application. * * @param applicationId an {@link ApplicationId} * @return session id of given application id. * @throws IllegalArgumentException if the application does not exist */ - public long getSessionIdForApplication(ApplicationId applicationId) { - String path = applicationPath(applicationId).getAbsolute(); - try { - return Long.parseLong(Utf8.toString(curator.framework().getData().forPath(path))); - } catch (Exception e) { - throw new IllegalArgumentException(TenantRepository.logPre(applicationId) + "Unable to read the session id from '" + path + "'", e); - } + public long requireActiveSessionOf(ApplicationId applicationId) { + return activeSessionOf(applicationId) + .orElseThrow(() -> new IllegalArgumentException("Application '" + applicationId + "' has no active session.")); } /** * Returns a transaction which deletes this application. */ - public CuratorTransaction deleteApplication(ApplicationId applicationId) { + public CuratorTransaction createDeleteTransaction(ApplicationId applicationId) { return CuratorTransaction.from(CuratorOperations.delete(applicationPath(applicationId).getAbsolute()), curator); } @@ -133,7 +144,7 @@ public class TenantApplications { * Removes all applications not known to this from the config server state. */ public void removeUnusedApplications() { - reloadHandler.removeApplicationsExcept(Set.copyOf(listApplications())); + reloadHandler.removeApplicationsExcept(Set.copyOf(activeApplications())); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index 6345532d4ff..cdf995b80bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -56,7 +56,7 @@ public class ListApplicationsHandler extends HttpHandler { Utils.checkThatTenantExists(tenantRepository, tenantName); Tenant tenant = tenantRepository.getTenant(tenantName); TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.listApplications(); + return applicationRepo.activeApplications(); } private static String createUrlStringFromId(String urlBase, ApplicationId id, Zone zone) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 0cdf5ebfe95..6a0a4a19737 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -64,6 +64,7 @@ public class LocalSession extends Session implements Comparable<LocalSession> { Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, Instant now) { + applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( Curator.CompletionWaiter waiter = zooKeeperClient.createPrepareWaiter(); ConfigChangeActions actions = sessionPreparer.prepare(sessionContext, logger, params, currentActiveApplicationSet, tenantPath, now); @@ -95,7 +96,7 @@ public class LocalSession extends Session implements Comparable<LocalSession> { zooKeeperClient.createActiveWaiter(); superModelGenerationCounter.increment(); // TODO jvenstad: I hope this counter isn't used for serious things, as it's updated way ahead of activation. Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutApplicationTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); + transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 15182813a22..ccd5684b9ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -190,9 +190,9 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { } private void loadSessionIfActive(RemoteSession session) { - for (ApplicationId applicationId : applicationRepo.listApplications()) { + for (ApplicationId applicationId : applicationRepo.activeApplications()) { try { - if (applicationRepo.getSessionIdForApplication(applicationId) == session.getSessionId()) { + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { log.log(LogLevel.DEBUG, "Found active application for session " + session.getSessionId() + " , loading it"); loadActiveSession(session); break; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 90eeb89dc8e..d8ba5890545 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -188,9 +188,9 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { } private long getActiveSessionId(ApplicationId applicationId) { - List<ApplicationId> applicationIds = applicationRepo.listApplications(); + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); if (applicationIds.contains(applicationId)) { - return applicationRepo.getSessionIdForApplication(applicationId); + return applicationRepo.requireActiveSessionOf(applicationId); } return nonExistingActiveSession; } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index b2c426e707b..5aca82409a9 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -76,115 +76,77 @@ <handler id='com.yahoo.vespa.config.server.http.HttpGetConfigHandler' bundle='configserver'> <binding>http://*/config/v1/*/*</binding> - <binding>https://*/config/v1/*/*</binding> <binding>http://*/config/v1/*</binding> - <binding>https://*/config/v1/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.HttpListConfigsHandler' bundle='configserver'> <binding>http://*/config/v1/</binding> - <binding>https://*/config/v1/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.HttpListNamedConfigsHandler' bundle='configserver'> <binding>http://*/config/v1/*/</binding> - <binding>https://*/config/v1/*/</binding> <binding>http://*/config/v1/*/*/</binding> - <binding>https://*/config/v1/*/*/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.status.StatusHandler' bundle='configserver'> <binding>http://*/status</binding> - <binding>https://*/status</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.flags.FlagsHandler' bundle='configserver'> <binding>http://*/flags/v1</binding> - <binding>https://*/flags/v1</binding> <binding>http://*/flags/v1/*</binding> - <binding>https://*/flags/v1/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.TenantHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/</binding> - <binding>https://*/application/v2/tenant/</binding> <binding>http://*/application/v2/tenant/*</binding> - <binding>https://*/application/v2/tenant/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionCreateHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session</binding> - <binding>https://*/application/v2/tenant/*/session</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionPrepareHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/prepared</binding> - <binding>https://*/application/v2/tenant/*/session/*/prepared</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionActiveHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/active</binding> - <binding>https://*/application/v2/tenant/*/session/*/active</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/prepareandactivate</binding> - <binding>https://*/application/v2/tenant/*/prepareandactivate</binding> <binding>http://*/application/v2/tenant/*/session/*/prepareandactivate</binding> - <binding>https://*/application/v2/tenant/*/session/*/prepareandactivate</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionContentHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/content/*</binding> - <binding>https://*/application/v2/tenant/*/session/*/content/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ListApplicationsHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/application/</binding> - <binding>https://*/application/v2/tenant/*/application/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ApplicationHandler' bundle='configserver'> <!-- WARNING: THIS LIST *MUST* MATCH THE ONE IN ApplicationHandler::getBindingMatch --> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*</binding> <binding>http://*/application/v2/tenant/*/application/*</binding> - <binding>https://*/application/v2/tenant/*/application/*</binding> <binding>http://*/application/v2/tenant/*/application/*/logs</binding> - <binding>https://*/application/v2/tenant/*/application/*/logs</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HttpGetConfigHandler' bundle='configserver'> <binding>http://*/config/v2/tenant/*/application/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/*</binding> <binding>http://*/config/v2/tenant/*/application/*/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/*/*</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler' bundle='configserver'> <binding>http://*/config/v2/tenant/*/application/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HttpListNamedConfigsHandler' bundle='configserver'> <binding>http://*/config/v2/tenant/*/application/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/*/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HostHandler' bundle='configserver'> <binding>http://*/application/v2/host/*</binding> - <binding>https://*/application/v2/host/*</binding> </handler> <http> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 5d356ddf88e..e0fa760b35d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -315,7 +315,7 @@ public class ApplicationRepositoryTest { assertTrue(deployment2.isPresent()); deployment2.get().activate(); // session 3 - long activeSessionId = tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId()); + long activeSessionId = tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId()); clock.advance(Duration.ofSeconds(10)); Optional<com.yahoo.config.provision.Deployment> deployment3 = tester.redeployFromLocalActive(); @@ -325,7 +325,7 @@ public class ApplicationRepositoryTest { LocalSession deployment3session = ((com.yahoo.vespa.config.server.deploy.Deployment) deployment3.get()).session(); assertNotEquals(activeSessionId, deployment3session); // No change to active session id - assertEquals(activeSessionId, tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId())); + assertEquals(activeSessionId, tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId())); assertEquals(3, tester.tenant().getLocalSessionRepo().listSessions().size()); clock.advance(Duration.ofHours(1)); // longer than session lifetime diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index a708e4d8ace..69c88dc0275 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -39,12 +39,12 @@ public class TenantApplicationsTest { writeApplicationData(createApplicationId("foo"), 3L); writeApplicationData(createApplicationId("bar"), 4L); TenantApplications repo = createZKAppRepo(); - List<ApplicationId> applications = repo.listApplications(); + List<ApplicationId> applications = repo.activeApplications(); assertThat(applications.size(), is(2)); - assertThat(applications.get(0).application().value(), is("foo")); - assertThat(applications.get(1).application().value(), is("bar")); - assertThat(repo.getSessionIdForApplication(applications.get(0)), is(3L)); - assertThat(repo.getSessionIdForApplication(applications.get(1)), is(4L)); + assertThat(applications.get(0).application().value(), is("bar")); + assertThat(applications.get(1).application().value(), is("foo")); + assertThat(repo.requireActiveSessionOf(applications.get(0)), is(4L)); + assertThat(repo.requireActiveSessionOf(applications.get(1)), is(3L)); } @Test @@ -52,7 +52,7 @@ public class TenantApplicationsTest { writeApplicationData(createApplicationId("foo"), 3L); writeApplicationData("invalid", 3L); TenantApplications repo = createZKAppRepo(); - List<ApplicationId> applications = repo.listApplications(); + List<ApplicationId> applications = repo.activeApplications(); assertThat(applications.size(), is(1)); assertThat(applications.get(0).application().value(), is("foo")); } @@ -60,7 +60,7 @@ public class TenantApplicationsTest { @Test(expected = IllegalArgumentException.class) public void require_that_requesting_session_for_unknown_application_throws_exception() throws Exception { TenantApplications repo = createZKAppRepo(); - repo.getSessionIdForApplication(createApplicationId("nonexistent")); + repo.requireActiveSessionOf(createApplicationId("nonexistent")); } @Test(expected = IllegalArgumentException.class) @@ -70,18 +70,19 @@ public class TenantApplicationsTest { curatorFramework.create().creatingParentsIfNeeded() .forPath(TenantRepository.getApplicationsPath(tenantName).append(baz.serializedForm()).getAbsolute()); TenantApplications repo = createZKAppRepo(); - repo.getSessionIdForApplication(baz); + repo.requireActiveSessionOf(baz); } @Test public void require_that_application_ids_can_be_written() throws Exception { TenantApplications repo = createZKAppRepo(); ApplicationId myapp = createApplicationId("myapp"); - repo.createPutApplicationTransaction(myapp, 3l).commit(); + repo.createApplication(myapp); + repo.createPutTransaction(myapp, 3l).commit(); String path = TenantRepository.getApplicationsPath(tenantName).append(myapp.serializedForm()).getAbsolute(); assertTrue(curatorFramework.checkExists().forPath(path) != null); assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("3")); - repo.createPutApplicationTransaction(myapp, 5l).commit(); + repo.createPutTransaction(myapp, 5l).commit(); assertTrue(curatorFramework.checkExists().forPath(path) != null); assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("5")); } @@ -91,13 +92,15 @@ public class TenantApplicationsTest { TenantApplications repo = createZKAppRepo(); ApplicationId id1 = createApplicationId("myapp"); ApplicationId id2 = createApplicationId("myapp2"); - repo.createPutApplicationTransaction(id1, 1).commit(); - repo.createPutApplicationTransaction(id2, 1).commit(); - assertThat(repo.listApplications().size(), is(2)); - repo.deleteApplication(id1).commit(); - assertThat(repo.listApplications().size(), is(1)); - repo.deleteApplication(id2).commit(); - assertThat(repo.listApplications().size(), is(0)); + repo.createApplication(id1); + repo.createApplication(id2); + repo.createPutTransaction(id1, 1).commit(); + repo.createPutTransaction(id2, 1).commit(); + assertThat(repo.activeApplications().size(), is(2)); + repo.createDeleteTransaction(id1).commit(); + assertThat(repo.activeApplications().size(), is(1)); + repo.createDeleteTransaction(id2).commit(); + assertThat(repo.activeApplications().size(), is(0)); } @Test @@ -108,7 +111,7 @@ public class TenantApplicationsTest { MockReloadHandler reloadHandler = new MockReloadHandler(); TenantApplications repo = createZKAppRepo(reloadHandler); assertNull(reloadHandler.lastRemoved); - repo.deleteApplication(foo).commit(); + repo.createDeleteTransaction(foo).commit(); long endTime = System.currentTimeMillis() + 60_000; while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { Thread.sleep(100); @@ -126,11 +129,7 @@ public class TenantApplicationsTest { } private static ApplicationId createApplicationId(String name) { - return new ApplicationId.Builder() - .tenant(tenantName.value()) - .applicationName(name) - .instanceName("myinst") - .build(); + return ApplicationId.from(tenantName.value(), name, "myinst"); } private void writeApplicationData(ApplicationId applicationId, long sessionId) throws Exception { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 7ffb9552cf8..6b67dcc4e9a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -183,7 +183,7 @@ public class DeployTester { public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() - .getSessionIdForApplication(applicationId)); + .requireActiveSessionOf(applicationId)); return session.getAllocatedHosts(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index b0bb3bf244f..c6a8e1f2f9d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -56,12 +56,14 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { session2 = new MockSession(2l, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); Tenant tenant1 = tenantRepository.getTenant(tenantName1); tenant1.getLocalSessionRepo().addSession(session2); - tenant1.getApplicationRepo().createPutApplicationTransaction(idTenant1, 2l).commit(); + tenant1.getApplicationRepo().createApplication(idTenant1); + tenant1.getApplicationRepo().createPutTransaction(idTenant1, 2l).commit(); MockSession session3 = new MockSession(3l, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); Tenant tenant2 = tenantRepository.getTenant(tenantName2); tenant2.getLocalSessionRepo().addSession(session3); - tenant2.getApplicationRepo().createPutApplicationTransaction(idTenant2, 3l).commit(); + tenant2.getApplicationRepo().createApplication(idTenant2); + tenant2.getApplicationRepo().createPutTransaction(idTenant2, 3l).commit(); handler = new ApplicationHandler(ApplicationHandler.testOnlyContext(), Zone.defaultZone(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index d3dcbcc2252..fa560ceae4d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -223,13 +223,13 @@ public class ApplicationHandlerTest { } private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { - long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().getSessionIdForApplication(applicationId); + long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); assertNull(tenantRepository.getTenant(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); } private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { - long sessionId = tenant.getApplicationRepo().getSessionIdForApplication(applicationId); + long sessionId = tenant.getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, true); assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 2d3dcc592f7..1db70956407 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -46,7 +46,8 @@ public class HostHandlerTest { private HostHandler hostHandler; static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId) { - tenant.getApplicationRepo().createPutApplicationTransaction(applicationId, sessionId).commit(); + tenant.getApplicationRepo().createApplication(applicationId); + tenant.getApplicationRepo().createPutTransaction(applicationId, sessionId).commit(); ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index f57e7f09b39..f97bc443a38 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -51,17 +51,17 @@ public class ListApplicationsHandlerTest { final String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; assertResponse(url, Response.Status.OK, "[]"); - applicationRepo.createPutApplicationTransaction( - new ApplicationId.Builder().tenant("tenant").applicationName("foo").instanceName("quux").build(), - 1).commit(); + ApplicationId id1 = ApplicationId.from("mytenant", "foo", "quux"); + applicationRepo.createApplication(id1); + applicationRepo.createPutTransaction(id1, 1).commit(); assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); - applicationRepo.createPutApplicationTransaction( - new ApplicationId.Builder().tenant("tenant").applicationName("bali").instanceName("quux").build(), - 1).commit(); + ApplicationId id2 = ApplicationId.from("mytenant", "bali", "quux"); + applicationRepo.createApplication(id2); + applicationRepo.createPutTransaction(id2, 1).commit(); assertResponse(url, Response.Status.OK, - "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"," + - "\"" + url + "bali/environment/dev/region/us-east/instance/quux\"]" + "[\"" + url + "bali/environment/dev/region/us-east/instance/quux\"," + + "\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]" ); } @@ -82,12 +82,12 @@ public class ListApplicationsHandlerTest { @Test public void require_that_listing_works_with_multiple_tenants() throws Exception { - applicationRepo.createPutApplicationTransaction(new ApplicationId.Builder() - .tenant("tenant") - .applicationName("foo").instanceName("quux").build(), 1).commit(); - applicationRepo2.createPutApplicationTransaction(new ApplicationId.Builder() - .tenant("tenant") - .applicationName("quux").instanceName("foo").build(), 1).commit(); + ApplicationId id1 = ApplicationId.from("mytenant", "foo", "quux"); + applicationRepo.createApplication(id1); + applicationRepo.createPutTransaction(id1, 1).commit(); + ApplicationId id2 = ApplicationId.from("foobar", "quux", "foo"); + applicationRepo2.createApplication(id2); + applicationRepo2.createPutTransaction(id2, 1).commit(); String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 380b76c30af..858d1e0eaa7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -323,6 +323,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); session = createRemoteSession(sessionId, initialStatus, zkClient); addLocalSession(sessionId, deployData, zkClient); + tenantRepository.getTenant(tenantName).getApplicationRepo().createApplication(ApplicationId.from(tenantName.value(), + deployData.getApplicationName(), + InstanceName.defaultName().value())); metaData = localRepo.getSession(sessionId).getMetaData(); actResponse = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); return this; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 927ac26b77d..0946ef3992c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -194,7 +194,8 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .applicationName("foo") .instanceName("quux") .build(); - applicationRepo.createPutApplicationTransaction(fooId, 2).commit(); + applicationRepo.createApplication(fooId); + applicationRepo.createPutTransaction(fooId, 2).commit(); assertFromParameter("3", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux"); localSessionRepo.addSession(new SessionHandlerTest.MockSession(5l, FilesApplicationPackage.fromFile(testApp))); ApplicationId bioId = new ApplicationId.Builder() @@ -202,7 +203,8 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .applicationName("foobio") .instanceName("quux") .build(); - applicationRepo.createPutApplicationTransaction(bioId, 5).commit(); + applicationRepo.createApplication(bioId); + applicationRepo.createPutTransaction(bioId, 5).commit(); assertFromParameter("6", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foobio/environment/staging/region/baz/instance/quux"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index e7db4dcf58f..a4432dcbfcd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -196,12 +196,14 @@ public class LocalSessionTest { zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); sessionDir.createNewFile(); + TenantApplications applications = TenantApplications.create(curator, new MockReloadHandler(), tenant); + applications.createApplication(zkc.readApplicationId()); return new LocalSession(tenant, sessionId, preparer, new SessionContext( FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, - TenantApplications.create(curator, new MockReloadHandler(), tenant), + applications, new HostRegistry<>(), superModelGenerationCounter, flagSource)); diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index f6bdeb02b9c..645c231531d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -6,12 +6,8 @@ import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import org.json.JSONException; -import org.json.JSONObject; -import java.io.IOException; import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; @@ -32,47 +28,19 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - JSONObject responseJSON = new JSONObject(); Instant earliestLogThreshold = Optional.ofNullable(request.getProperty("from")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); Instant latestLogThreshold = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now); - try { - if (request.hasProperty("streaming")) { - return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } - - @Override - public void render(OutputStream outputStream) { - logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); - } - }; - } - - JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold); - responseJSON.put("logs", logJson); - } catch (IOException | JSONException e) { - return new HttpResponse(404) { - @Override - public void render(OutputStream outputStream) {} - }; - } - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream); - outputStreamWriter.write(responseJSON.toString()); - outputStreamWriter.close(); + { + headers().add("Content-Encoding", "gzip"); } - @Override - public String getContentType() { - return "application/json"; + public void render(OutputStream outputStream) { + logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); } }; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 663741f9bef..6cb92244522 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -3,10 +3,7 @@ package com.yahoo.container.handler; import com.yahoo.collections.Pair; import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; @@ -16,15 +13,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.time.Duration; import java.time.Instant; -import java.util.Base64; import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; class LogReader { @@ -41,19 +35,6 @@ class LogReader { this.logFilePattern = logFilePattern; } - // TODO: Remove when all clients use streaming - JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException { - JSONObject json = new JSONObject(); - latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified - for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { - StringBuilder filenameBuilder = new StringBuilder(); - logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString())); - byte[] fileData = file.toString().endsWith(".gz") ? new GZIPInputStream(new ByteArrayInputStream(Files.readAllBytes(file))).readAllBytes() : Files.readAllBytes(file); - json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(fileData)); - } - return json; - } - void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { try { for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index d578a745c9e..f0885451431 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -3,12 +3,11 @@ package com.yahoo.container.handler; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.time.Instant; import java.util.concurrent.Executor; @@ -17,7 +16,6 @@ import static org.mockito.Mockito.mock; public class LogHandlerTest { - @Test public void handleCorrectlyParsesQueryParameters() throws IOException { MockLogReader mockLogReader = new MockLogReader(); @@ -28,7 +26,7 @@ public class LogHandlerTest { HttpResponse response = logHandler.handle(HttpRequest.createTestRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.GET)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); response.render(bos); - String expectedResponse = "{\"logs\":{\"one\":\"newer_log\"}}"; + String expectedResponse = "newer log"; assertEquals(expectedResponse, bos.toString()); } @@ -37,7 +35,7 @@ public class LogHandlerTest { HttpResponse response = logHandler.handle(HttpRequest.createTestRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.GET)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); response.render(bos); - String expectedResponse = "{\"logs\":{\"two\":\"older_log\"}}"; + String expectedResponse = "older log"; assertEquals(expectedResponse, bos.toString()); } @@ -49,12 +47,14 @@ public class LogHandlerTest { } @Override - protected JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws JSONException { - if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { - return new JSONObject("{\"one\":\"newer_log\"}"); - } else { - return new JSONObject("{\"two\":\"older_log\"}"); - } + protected void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { + try { + if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { + outputStream.write("newer log".getBytes()); + } else { + outputStream.write("older log".getBytes()); + } + } catch (Exception e) {} } } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 464d6f772eb..27300e5f3f5 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -2,7 +2,6 @@ package com.yahoo.container.handler; import com.yahoo.vespa.test.file.TestFileSystem; -import org.json.JSONObject; import org.junit.Before; import org.junit.Test; @@ -15,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.util.Scanner; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -37,47 +35,37 @@ public class LogReaderTest { Files.setLastModifiedTime( Files.write(logDirectory.resolve("subfolder/log2.log"), "This is another log file\n".getBytes()), FileTime.from(Instant.ofEpochMilli(234))); - } - @Test - public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log.gz\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}"; - String actual = json.toString(); - assertEquals(expected, actual); } @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - JSONObject json = logReader.readLogs(Instant.MAX, Instant.MIN); - String expected = "{}"; - String actual = json.toString(); + logReader.writeLogs(baos, Instant.ofEpochMilli(235), Instant.ofEpochMilli(300)); + String expected = ""; + String actual = decompress(baos.toByteArray()); assertEquals(expected, actual); } @Test public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); - JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\"}"; - String actual = json.toString(); + logReader.writeLogs(baos, Instant.ofEpochMilli(21), Instant.now()); + String expected = "This is another log file\n"; + String actual = decompress(baos.toByteArray()); assertEquals(expected, actual); } @Test public void testZippedStreaming() throws IOException { - ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now()); - GZIPInputStream unzippedIs = new GZIPInputStream(new ByteArrayInputStream(zippedBaos.toByteArray())); - - Scanner s = new Scanner(unzippedIs).useDelimiter("\\A"); - String actual = s.hasNext() ? s.next() : ""; String expected = "This is one log file\nThis is another log file\n"; + String actual = decompress(zippedBaos.toByteArray()); assertEquals(expected, actual); } @@ -89,4 +77,10 @@ public class LogReaderTest { return baos.toByteArray(); } + private String decompress(byte[] input) throws IOException { + if (input.length == 0) return ""; + byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); + return new String(decompressed); + } + } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 885b157b4c3..b968d1c391f 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5066,6 +5066,7 @@ "public void addItem(com.yahoo.prelude.query.Item)", "public void addItem(int, com.yahoo.prelude.query.Item)", "public boolean isEmpty()", + "public com.yahoo.prelude.query.Item and(com.yahoo.prelude.query.Item)", "public static java.util.List getPositiveTerms(com.yahoo.prelude.query.Item)", "public bridge synthetic com.yahoo.prelude.query.CompositeItem clone()", "public bridge synthetic com.yahoo.prelude.query.Item clone()", diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java index e87c94fa3be..ee64632d40a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/SortDataHitSorter.java @@ -20,13 +20,13 @@ public class SortDataHitSorter { Collections.sort(hits, getComparator(sorting, fallbackComparator)); } - public static boolean isSortable(Hit hit, HitGroup hitGroup) { - if (hitGroup.getQuery() == null) { + public static boolean isSortable(Hit hit, Sorting sorting) { + if (sorting == null) { return false; } if (hit instanceof FastHit) { var fhit = (FastHit) hit; - return fhit.hasSortData(hitGroup.getQuery().getRanking().getSorting()); + return fhit.hasSortData(sorting); } else { return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 499cacd89c5..5e994dac5d6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -13,8 +13,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; /** * Parser for queries of type all. * - * @author Steinar Knutsen - * @author bratseth + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ public class AllParser extends SimpleParser { @@ -33,37 +32,37 @@ public class AllParser extends SimpleParser { protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and and'ed items, then combine. - AndItem and = null; - NotItem not = null; // Store negatives here as we go + AndItem and=null; + NotItem not=null; // Store negatives here as we go Item current; // Find all items do { - current = negativeItem(); - if (current != null) { - not = addNot(current, not); + current=negativeItem(); + if (current!=null) { + not=addNot(current,not); continue; } - current = positiveItem(); - if (current == null) + current=positiveItem(); + if (current==null) current = indexableItem(); if (current == null) current = compositeItem(); - if (current != null) - and = addAnd(current, and); + if (current!=null) + and=addAnd(current,and); if (current == null) tokens.skip(); } while (tokens.hasNext()); // Combine the items - Item topLevel = and; + Item topLevel=and; - if (not != null && topLevel != null) { + if (not!=null && topLevel!=null) { not.setPositiveItem(topLevel); - topLevel = not; + topLevel=not; } return simplifyUnnecessaryComposites(topLevel); @@ -79,23 +78,23 @@ public class AllParser extends SimpleParser { return root.getRoot() instanceof NullItem ? null : root.getRoot(); } - protected AndItem addAnd(Item item, AndItem and) { - if (and == null) - and = new AndItem(); + protected AndItem addAnd(Item item,AndItem and) { + if (and==null) + and=new AndItem(); and.addItem(item); return and; } protected OrItem addOr(Item item,OrItem or) { - if (or == null) - or = new OrItem(); + if (or==null) + or=new OrItem(); or.addItem(item); return or; } protected NotItem addNot(Item item,NotItem not) { - if (not == null) - not = new NotItem(); + if (not==null) + not=new NotItem(); not.addNegativeItem(item); return not; } @@ -104,7 +103,8 @@ public class AllParser extends SimpleParser { int position = tokens.getPosition(); Item item = null; try { - if ( ! tokens.skip(MINUS)) return null; + if (!tokens.skipMultiple(MINUS)) return null; + if (tokens.currentIsNoIgnore(SPACE)) return null; item = indexableItem(); @@ -122,15 +122,8 @@ public class AllParser extends SimpleParser { } } } - if (item != null) + if (item!=null) item.setProtected(true); - - // Heuristic overdrive engaged! - // Interpret -N as a positive item matching a negative number (by backtracking out of this) - // but interpret --N as a negative item matching a negative number - if ( item instanceof IntItem && ! ((IntItem)item).getNumber().startsWith(("-"))) - item = null; - return item; } finally { if (item == null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 9ddfea6dffb..291beb40b4c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -10,10 +10,10 @@ import static com.yahoo.prelude.query.parser.Token.Kind.PLUS; import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; /** - * Base class for parsers of the "simple" query languages (query types ANY and ALL). + * Base class for parsers of the "simple" query languages (query types + * ANY and ALL). * * @author Steinar Knutsen - * @author bratseth */ abstract class SimpleParser extends StructuredParser { @@ -25,6 +25,7 @@ abstract class SimpleParser extends StructuredParser { return anyItems(false); // Nesteds are any even if all on top level } + protected abstract Item negativeItem(); /** @@ -162,7 +163,11 @@ abstract class SimpleParser extends StructuredParser { return false; } - /** Removes and returns the first <i>not</i> found in the composite, or returns null if there's none */ + + /** + * Removes and returns the first <i>not</i> found in the composite, + * or returns null if there's none + */ private NotItem removeNot(CompositeItem composite) { for (int i = 0; i < composite.getItemCount(); i++) { if (composite.getItem(i) instanceof NotItem) { @@ -179,7 +184,7 @@ abstract class SimpleParser extends StructuredParser { Item item = null; try { - if ( ! tokens.skipMultiple(PLUS)) { + if (!tokens.skipMultiple(PLUS)) { return null; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 8ecd4d8f81c..ec1f79828c1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -60,7 +60,7 @@ abstract class StructuredParser extends AbstractParser { String indexName = indexPrefix(); setSubmodeFromIndex(indexName, indexFacts); - item = number(); + item = number(indexName != null); if (item == null) { item = phrase(); @@ -147,36 +147,39 @@ abstract class StructuredParser extends AbstractParser { List<Token> firstWord = new ArrayList<>(); List<Token> secondWord = new ArrayList<>(); - tokens.skip(LSQUAREBRACKET); + tokens.skip(LSQUAREBRACKET); // For test 93 and 60 - if ( ! tokens.currentIs(WORD) && ! tokens.currentIs(NUMBER) && ! tokens.currentIs(UNDERSCORE)) { + if (!tokens.currentIs(WORD) && !tokens.currentIs(NUMBER) + && !tokens.currentIs(UNDERSCORE)) { return null; } firstWord.add(tokens.next()); while (tokens.currentIsNoIgnore(UNDERSCORE) - || tokens.currentIsNoIgnore(WORD) - || tokens.currentIsNoIgnore(NUMBER)) { + || tokens.currentIsNoIgnore(WORD) + || tokens.currentIsNoIgnore(NUMBER)) { firstWord.add(tokens.next()); } if (tokens.currentIsNoIgnore(DOT)) { tokens.skip(); - if (tokens.currentIsNoIgnore(WORD) || tokens.currentIsNoIgnore(NUMBER)) { + if (tokens.currentIsNoIgnore(WORD) + || tokens.currentIsNoIgnore(NUMBER)) { secondWord.add(tokens.next()); } else { return null; } while (tokens.currentIsNoIgnore(UNDERSCORE) - || tokens.currentIsNoIgnore(WORD) - || tokens.currentIsNoIgnore(NUMBER)) { + || tokens.currentIsNoIgnore(WORD) + || tokens.currentIsNoIgnore(NUMBER)) { secondWord.add(tokens.next()); } } - if ( ! tokens.skipNoIgnore(COLON)) + if (!tokens.skipNoIgnore(COLON)) { return null; + } if (secondWord.size() == 0) { item = concatenate(firstWord); @@ -192,7 +195,8 @@ abstract class StructuredParser extends AbstractParser { return null; } else { if (nothingAhead(false)) { - // correct index syntax, correct name, but followed by noise. Let's skip this. + // correct index syntax, correct name, but followed + // by noise. Let's skip this. nothingAhead(true); position = tokens.getPosition(); item = indexPrefix(); @@ -249,11 +253,11 @@ abstract class StructuredParser extends AbstractParser { private boolean endOfNumber() { return tokens.currentIsNoIgnore(SPACE) - || tokens.currentIsNoIgnore(RSQUAREBRACKET) - || tokens.currentIsNoIgnore(SEMICOLON) - || tokens.currentIsNoIgnore(RBRACE) - || tokens.currentIsNoIgnore(EOF) - || tokens.currentIsNoIgnore(EXCLAMATION); + || tokens.currentIsNoIgnore(RSQUAREBRACKET) + || tokens.currentIsNoIgnore(SEMICOLON) + || tokens.currentIsNoIgnore(RBRACE) + || tokens.currentIsNoIgnore(EOF) + || tokens.currentIsNoIgnore(EXCLAMATION); } private String decimalPart() { @@ -273,19 +277,19 @@ abstract class StructuredParser extends AbstractParser { } } - private IntItem number() { + private IntItem number(boolean hasIndex) { int position = tokens.getPosition(); IntItem item = null; try { - item = numberRange(); + if (item == null) { + item = numberRange(); + } - tokens.skip(LSQUAREBRACKET); - if (item == null) - tokens.skipNoIgnore(SPACE); + tokens.skip(LSQUAREBRACKET); // For test 93 and 60 // TODO: Better definition of start and end of numeric items - if (item == null && tokens.currentIsNoIgnore(MINUS) && (tokens.currentNoIgnore(1).kind == NUMBER)) { + if (item == null && hasIndex && tokens.currentIsNoIgnore(MINUS) && (tokens.currentNoIgnore(1).kind == NUMBER)) { tokens.skipNoIgnore(); Token t = tokens.next(); item = new IntItem("-" + t.toString() + decimalPart(), true); @@ -303,7 +307,7 @@ abstract class StructuredParser extends AbstractParser { if (item == null) { item = numberGreater(); } - if (item != null && ! endOfNumber()) { + if (item != null && !endOfNumber()) { item = null; } return item; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java index fbaf1675ff1..42cef67f189 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; + import java.util.List; + /** * An iterator-like view of a list of tokens, but typed, random-accessible * and with more convenience methods @@ -181,7 +183,8 @@ final class TokenPosition { /** * Skips one or zero items of the given kind. * - * @return true if one item was skipped, false if none was, or if there are no more tokens + * @return true if one item was skipped, false if none was, + * or if there are no more tokens */ public boolean skip(Token.Kind kind) { Token current = current(); @@ -195,16 +198,20 @@ final class TokenPosition { } /** - * Skips one or zero items of the given kind, without ignoring spaces + * Skips one or zero items of the given kind, without ignoring + * spaces * - * @return true if one item was skipped, false if none was or if there are no more tokens + * @return true if one item was skipped, false if none was, + * or if there are no more tokens */ public boolean skipNoIgnore(Token.Kind kind) { Token current = currentNoIgnore(); - if (current == null || current.kind != kind) return false; + if (current == null || current.kind != kind) { + return false; + } - skipNoIgnore(); + skip(); return true; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 5a8102ff5c7..033b507b4a4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -154,14 +154,15 @@ public class ProtobufSerialization { return convertFromResult(searchResult).toByteArray(); } - public static Result deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher) + public static Result deserializeToSearchResult(byte[] payload, Query query, VespaBackEndSearcher searcher, int partId, int distKey) throws InvalidProtocolBufferException { var protobuf = SearchProtocol.SearchReply.parseFrom(payload); - var result = convertToResult(query, protobuf, searcher.getDocumentDatabase(query)); + var result = convertToResult(query, protobuf, searcher.getDocumentDatabase(query), partId, distKey, searcher.getName()); return result; } - private static Result convertToResult(Query query, SearchProtocol.SearchReply protobuf, DocumentDatabase documentDatabase) { + private static Result convertToResult(Query query, SearchProtocol.SearchReply protobuf, DocumentDatabase documentDatabase, int partId, + int distKey, String source) { var result = new Result(query); result.setTotalHitCount(protobuf.getTotalHitCount()); @@ -188,12 +189,20 @@ public class ProtobufSerialization { hit.setRelevance(new Relevance(replyHit.getRelevance())); hit.setGlobalId(new GlobalId(replyHit.getGlobalId().toByteArray())); - hit.setSortData(replyHit.getSortData().toByteArray(), sorting); + if (!replyHit.getSortData().isEmpty()) { + hit.setSortData(replyHit.getSortData().toByteArray(), sorting); + } hit.setFillable(); hit.setCached(false); + hit.setPartId(partId); + hit.setDistributionKey(distKey); + hit.setSource(source); result.hits().add(hit); } + if(sorting != null) { + result.hits().setSorted(true); + } return result; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 75e9b06f445..d118d1835a8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -3,7 +3,6 @@ package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; -import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -86,15 +85,7 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe CompressionType compression = CompressionType.valueOf(protobufResponse.compression()); byte[] payload = resourcePool.compressor().decompress(protobufResponse.compressedPayload(), compression, protobufResponse.uncompressedSize()); - var result = ProtobufSerialization.deserializeToSearchResult(payload, query, searcher); - result.hits().unorderedIterator().forEachRemaining(hit -> { - if(hit instanceof FastHit) { - FastHit fhit = (FastHit) hit; - fhit.setPartId(node.pathIndex()); - fhit.setDistributionKey(node.key()); - } - hit.setSource(getName()); - }); + var result = ProtobufSerialization.deserializeToSearchResult(payload, query, searcher, node.pathIndex(), node.key()); return result; } diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index bacfe8a949a..6eba6ae4837 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -108,9 +108,12 @@ public class QueryTree extends CompositeItem { // -------------- Facade - /** Modifies this query to become the current query AND the given item */ - // TODO: Make sure this is complete, unit test and make it public - private void and(Item item) { + /** + * Modifies this query to become the current query AND the given item. + * + * @return the resulting root item in this + */ + public Item and(Item item) { if (isEmpty()) { setRoot(item); } @@ -126,12 +129,16 @@ public class QueryTree extends CompositeItem { notItem.addPositiveItem(getRoot()); setRoot(notItem); } + else if (getRoot() instanceof AndItem) { + ((AndItem) getRoot()).addItem(item); + } else { AndItem andItem = new AndItem(); andItem.addItem(getRoot()); andItem.addItem(item); setRoot(andItem); } + return getRoot(); } /** Returns a flattened list of all positive query terms under the given item */ diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java index b3d79f65df4..32c386f0e32 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java @@ -15,7 +15,8 @@ public interface Parser { * {@link QueryTree}. If parsing fails without an exception, the contained * root will be an instance of {@link com.yahoo.prelude.query.NullItem}. * - * @param query the Parsable to parse + * @param query + * the Parsable to parse * @return the parsed QueryTree, never null */ QueryTree parse(Parsable query); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java index 21bc22d7002..f77301f587c 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java @@ -20,7 +20,8 @@ import static com.yahoo.yolean.Exceptions.toMessageString; /** * Searcher that builds a PredicateItem from the &boolean properties and inserts it into a query. - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + * + * @author Magnar Nedland */ @After({ STEMMING, ACCENT_REMOVAL }) @Provides(BooleanSearcher.PREDICATE) @@ -74,7 +75,7 @@ public class BooleanSearcher extends Searcher { item.setIndexName(fieldName); new PredicateValueAttributeParser(item).parse(attributes); new PredicateRangeAttributeParser(item).parse(rangeAttributes); - QueryTreeUtil.andQueryItemWithRoot(query, item); + query.getModel().getQueryTree().and(item); } static public class PredicateValueAttributeParser extends BooleanAttributeParser { diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java index e4841ae6bd1..759c8ba1ee4 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java @@ -3,6 +3,7 @@ package com.yahoo.search.querytransform; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -10,27 +11,27 @@ import com.yahoo.search.query.QueryTree; * Utility class for manipulating a QueryTree. * * @author geirst + * @deprecated use QueryTree.and instead // TODO: Remove on Vespa 8 */ +@Deprecated public class QueryTreeUtil { - static public void andQueryItemWithRoot(Query query, Item item) { - andQueryItemWithRoot(query.getModel().getQueryTree(), item); + /** + * Adds the given item to this query + * + * @return the new root of the query tree + */ + static public Item andQueryItemWithRoot(Query query, Item item) { + return andQueryItemWithRoot(query.getModel().getQueryTree(), item); } - static public void andQueryItemWithRoot(QueryTree tree, Item item) { - if (tree.isEmpty()) { - tree.setRoot(item); - } else { - Item oldRoot = tree.getRoot(); - if (oldRoot.getClass() == AndItem.class) { - ((AndItem) oldRoot).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(oldRoot); - newRoot.addItem(item); - tree.setRoot(newRoot); - } - } + /** + * Adds the given item to this query + * + * @return the new root of the query tree + */ + static public Item andQueryItemWithRoot(QueryTree tree, Item item) { + return tree.and(item); } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java index cda41f5f62e..0b1387a16a2 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java @@ -15,7 +15,6 @@ import com.yahoo.text.MapParser; import java.util.LinkedHashMap; import java.util.Map; -import static com.yahoo.container.protect.Error.UNSPECIFIED; import com.yahoo.yolean.Exceptions; /** @@ -147,7 +146,7 @@ public class WandSearcher extends Searcher { InputResolver inputs = new InputResolver(query, execution); if ( ! inputs.hasValidData()) return execution.search(query); - QueryTreeUtil.andQueryItemWithRoot(query, createWandQueryItem(inputs)); + query.getModel().getQueryTree().and(createWandQueryItem(inputs)); query.trace("WandSearcher: Added WAND operator", true, 4); return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index fdebe78c20b..8755d0b0897 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -11,6 +11,7 @@ import com.yahoo.processing.response.ArrayDataList; import com.yahoo.processing.response.DataList; import com.yahoo.processing.response.DefaultIncomingData; import com.yahoo.processing.response.IncomingData; +import com.yahoo.search.query.Sorting; import java.util.ArrayList; import java.util.Collections; @@ -687,7 +688,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< hit.setAddNumber(size()); } - if (SortDataHitSorter.isSortable(hit, this)) { + if (SortDataHitSorter.isSortable(hit, currentSorting())) { hitsWithSortData++; } @@ -739,7 +740,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< errorHit = null; } - if (SortDataHitSorter.isSortable(hit, this)) { + if (SortDataHitSorter.isSortable(hit, currentSorting())) { hitsWithSortData--; } @@ -758,7 +759,7 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< if (!hit.isCached()) notCachedCount++; - if (SortDataHitSorter.isSortable(hit, this)) { + if (SortDataHitSorter.isSortable(hit, currentSorting())) { hitsWithSortData++; } } @@ -927,6 +928,15 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< return Iterables.filter(hits, isFillable); } + private Sorting currentSorting() { + var query = getQuery(); + if(query == null) { + return null; + } else { + return query.getRanking().getSorting(); + } + } + /** Returns the incoming hit buffer to which new hits can be added to this asynchronous, if supported by the instance */ @Override public IncomingData<Hit> incoming() { return incomingHits; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 3eac1d88784..af095fefc1c 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -712,7 +712,7 @@ public class YqlParser implements Parser { .setLanguage(language) .setDefaultIndexName(defaultIndex)).getRoot(); // the null check should be unnecessary, but is there to avoid having to suppress null warnings - if ( ! allowNullItem && (item == null || item instanceof NullItem)) + if ( !allowNullItem && (item == null || item instanceof NullItem)) throw new IllegalArgumentException("Parsing '" + wordData + "' only resulted in NullItem."); if (language != Language.ENGLISH) // mark the language used, unless it's the default diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 73f2ae7eb87..12f9ef2b18f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -1957,12 +1957,7 @@ public class ParseTestCase { @Test public void testNumbersAndNot() { - tester.assertParsed("AND a -12", "a -12", Query.Type.ALL); - } - - @Test - public void testNumbersAndDoubleNot() { - tester.assertParsed("+a --12", "a --12", Query.Type.ALL); + tester.assertParsed("+a -12", "a -12", Query.Type.ALL); } @Test @@ -1972,7 +1967,7 @@ public class ParseTestCase { @Test public void testSingleNegativeNumberLikeTerm() { - tester.assertParsed("-12", "-12", Query.Type.ALL); + tester.assertParsed(null, "-12", Query.Type.ALL); } @Test @@ -2009,12 +2004,7 @@ public class ParseTestCase { @Test public void testDecimalNumbersAndNot() { - tester.assertParsed("AND a -12.2", "a -12.2", Query.Type.ALL); - } - - @Test - public void testDecimalNumbersAndDoubleNot() { - tester.assertParsed("+a --12.2", "a --12.2", Query.Type.ALL); + tester.assertParsed("+a -12.2", "a -12.2", Query.Type.ALL); } @Test @@ -2024,7 +2014,7 @@ public class ParseTestCase { @Test public void testSingleNegativeDecimalNumberLikeTerm() { - tester.assertParsed("-12.2", "-12.2", Query.Type.ALL); + tester.assertParsed(null, "-12.2", Query.Type.ALL); } @Test @@ -2331,12 +2321,12 @@ public class ParseTestCase { @Test public void testSingleNegativeNumberLikeTermWeb() { - tester.assertParsed("-12", "-12", Query.Type.WEB); + tester.assertParsed(null, "-12", Query.Type.WEB); } @Test public void testSingleNegativeDecimalNumberLikeTermWeb() { - tester.assertParsed("-12.2", "-12.2", Query.Type.WEB); + tester.assertParsed(null, "-12.2", Query.Type.WEB); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java index 3cc3257194c..b6b7a1f5819 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MockClient.java @@ -83,7 +83,8 @@ public class MockClient implements Client { } } byte[] slimeBytes = BinaryFormat.encode(responseSlime); - Compressor.Compression compressionResult = compressor.compress(compression, slimeBytes); + CompressionType responseCompressionType = compression == CompressionType.INCOMPRESSIBLE ? CompressionType.NONE : compression; + Compressor.Compression compressionResult = compressor.compress(responseCompressionType, slimeBytes); GetDocsumsResponse response = new GetDocsumsResponse(compressionResult.type().getCode(), slimeBytes.length, compressionResult.data(), hitsContext); responseReceiver.receive(ResponseOrError.fromResponse(response)); diff --git a/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java new file mode 100644 index 00000000000..f929e54fd2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java @@ -0,0 +1,26 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.WordItem; +import org.junit.Assert; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class QueryTreeTest { + + @Test + public void testAddQueryItemWithRoot() { + Assert.assertEquals("AND a b", + new QueryTree(new WordItem("a")).and(new WordItem("b")).toString()); + + NotItem not = new NotItem(); + not.addNegativeItem(new WordItem("b")); + assertEquals("+a -b", + new QueryTree(new WordItem("a")).and(not).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java index c7a44e8aceb..720dcd0c4bc 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java @@ -6,7 +6,9 @@ import com.yahoo.prelude.query.Item; import com.yahoo.search.Result; public class TestUtils { + public static Item getQueryTreeRoot(Result result) { return result.getQuery().getModel().getQueryTree().getRoot(); } + } diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 6173d710434..b5c4166e4de 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -118,17 +118,6 @@ public class UserInputTestCase { } @Test - public void testNegativeNumberComparison() { - URIBuilder builder = searchUri(); - builder.setParameter("myinput", "-5"); - builder.setParameter("yql", - "select * from ecitem where rank(([{\"defaultIndex\":\"myfield\"}](userInput(@myinput))));"); - Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from ecitem where rank(myfield = (-5));", query.yqlRepresentation()); - assertEquals("RANK myfield:-5", query.getModel().getQueryTree().getRoot().toString()); - } - - @Test public void testAnnotatedUserInputUnrankedTerms() { URIBuilder builder = searchUri(); builder.setParameter("yql", diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java index 04dd91670c1..5b538f23fbd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/DeploymentId.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.identifiers; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Objects; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java index 849912ef581..32e94625c5a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Map; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java index aba3b5f3ab7..fffa849f7d3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; @@ -10,7 +11,7 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; */ public interface AthenzClientFactory { - AthenzService getControllerIdentity(); + AthenzIdentity getControllerIdentity(); ZmsClient createZmsClient(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index b43cb176607..91a9b5472f5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.IOException; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index d166bb0d3fb..d618464fc2a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -31,10 +31,14 @@ public class Node { private final long wantedRestartGeneration; private final long rebootGeneration; private final long wantedRebootGeneration; + private final String canonicalFlavor; + private final String clusterId; + private final ClusterType clusterType; public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState, - long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration) { + long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, + String canonicalFlavor, String clusterId, ClusterType clusterType) { this.hostname = hostname; this.state = state; this.type = type; @@ -48,13 +52,17 @@ public class Node { this.wantedRestartGeneration = wantedRestartGeneration; this.rebootGeneration = rebootGeneration; this.wantedRebootGeneration = wantedRebootGeneration; + this.canonicalFlavor = canonicalFlavor; + this.clusterId = clusterId; + this.clusterType = clusterType; } @TestOnly public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion) { this(hostname, state, type, owner, currentVersion, wantedVersion, - Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0); + Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0, + "d-2-8-50", "cluster", ClusterType.container); } public HostName hostname() { @@ -107,6 +115,18 @@ public class Node { return wantedRebootGeneration; } + public String canonicalFlavor() { + return canonicalFlavor; + } + + public String clusterId() { + return clusterId; + } + + public ClusterType clusterType() { + return clusterType; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -139,4 +159,11 @@ public class Node { unorchestrated } + /** Known cluster types. */ + public enum ClusterType { + admin, + container, + content + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 7b46e1f589f..95544c23db5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.List; import java.util.Set; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ServiceConvergence.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ServiceConvergence.java index 6cfdc9fadc8..b23faa2ec3f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ServiceConvergence.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ServiceConvergence.java @@ -4,10 +4,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.List; -import java.util.OptionalLong; /** * Service convergence status for an application. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java index 13b16fc4cd1..890059b86b2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; /** * A repository of application build artifacts. diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 1dd9da6dc32..f2334d01716 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -5,7 +5,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Optional; import java.util.stream.Stream; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java index e7d7cfe620d..c377b68bb22 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/AliasTarget.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Objects; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java index f47f2061000..a4219569ab1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/Record.java @@ -50,7 +50,10 @@ public class Record implements Comparable<Record> { PTR, SOA, SRV, - TXT + TXT, + SPF, + NAPTR, + CAA, } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java index 8151cce3d76..4b495ebf331 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.noderepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.io.IOException; import java.util.Collection; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMarketplace.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMarketplace.java new file mode 100644 index 00000000000..bb5c3749289 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMarketplace.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Marketplace; + +/** + * @author jonmv + */ +public class MockMarketplace implements Marketplace { + + @Override + public BillingInfo resolveCustomer(String registrationToken) { + return new BillingInfo("customer", "Vespa"); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java new file mode 100644 index 00000000000..b6524deb4d1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; +import com.yahoo.vespa.hosted.controller.api.role.Role; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author jonmv + */ +public class MockUserManagement implements UserManagement { + + private final Map<Role, Set<UserId>> memberships = new HashMap<>(); + + @Override + public void createRole(Role role) { + if (memberships.containsKey(role)) + throw new IllegalArgumentException(role + " already exists."); + + memberships.put(role, new HashSet<>()); + } + + @Override + public void deleteRole(Role role) { + memberships.remove(role); + } + + @Override + public void addUsers(Role role, Collection<UserId> users) { + memberships.get(role).addAll(users); + } + + @Override + public void removeUsers(Role role, Collection<UserId> users) { + memberships.get(role).removeAll(users); + } + + @Override + public List<UserId> listUsers(Role role) { + return List.copyOf(memberships.get(role)); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleId.java deleted file mode 100644 index 199f233835f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleId.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.yahoo.vespa.hosted.controller.api.integration.user; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; -import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.Roles; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; - -import java.util.Objects; - -/** - * An identifier for a role which users identified by {@link UserId}s can be members of, corresponding to a bound {@link Role}. - * - * @author jonmv - */ -public class RoleId { - - private final String value; - - private RoleId(String value) { - if (value.isBlank()) - throw new IllegalArgumentException("Id value must be non-blank."); - this.value = value; - } - - public static RoleId fromRole(TenantRole role) { - return new RoleId(valueOf(role)); - } - - public static RoleId fromRole(ApplicationRole role) { - return new RoleId(valueOf(role)); - } - - public static RoleId fromValue(String value) { - return new RoleId(value); - } - - public String value() { - return value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RoleId id = (RoleId) o; - return Objects.equals(value, id.value); - } - - @Override - public int hashCode() { - return Objects.hash(value); - } - - @Override - public String toString() { - return "role '" + value + "'"; - } - - /** Returns the {@link Role} this represent. */ - public Role toRole(Roles roles) { - String[] parts = value.split("\\."); - if (parts.length == 2) switch (parts[1]) { - case "tenantOwner": return roles.tenantOwner(TenantName.from(parts[0])); - case "tenantAdmin": return roles.tenantAdmin(TenantName.from(parts[0])); - case "tenantOperator": return roles.tenantOperator(TenantName.from(parts[0])); - } - if (parts.length == 3) switch (parts[2]) { - case "applicationOwner": return roles.applicationOwner(TenantName.from(parts[0]), ApplicationName.from(parts[1])); - case "applicationAdmin": return roles.applicationAdmin(TenantName.from(parts[0]), ApplicationName.from(parts[1])); - case "applicationOperator": return roles.applicationOperator(TenantName.from(parts[0]), ApplicationName.from(parts[1])); - case "applicationDeveloper": return roles.applicationDeveloper(TenantName.from(parts[0]), ApplicationName.from(parts[1])); - case "applicationReader": return roles.applicationReader(TenantName.from(parts[0]), ApplicationName.from(parts[1])); - } - throw new IllegalArgumentException("Malformed or illegal role value '" + value + "'."); - } - - private static String valueOf(TenantRole role) { - return valueOf(role.tenant()) + "." + valueOf(role.definition()); - } - - private static String valueOf(ApplicationRole role) { - return valueOf(role.tenant()) + "." + valueOf(role.application()) + "." + valueOf(role.definition()); - } - - private static String valueOf(TenantName tenant) { - if (tenant.value().contains(".")) - throw new IllegalArgumentException("Tenant names may not contain '.'."); - - return tenant.value(); - } - - private static String valueOf(ApplicationName application) { - if (application.value().contains(".")) - throw new IllegalArgumentException("Application names may not contain '.'."); - - return application.value(); - } - - private static String valueOf(RoleDefinition role) { - switch (role) { - case tenantOwner: return "tenantOwner"; - case tenantAdmin: return "tenantAdmin"; - case tenantOperator: return "tenantOperator"; - case applicationOwner: return "applicationOwner"; - case applicationAdmin: return "applicationAdmin"; - case applicationOperator: return "applicationOperator"; - case applicationDeveloper: return "applicationDeveloper"; - case applicationReader: return "applicationReader"; - default: throw new IllegalArgumentException("No value defined for role '" + role + "'."); - } - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java index c78dcc76854..ff25e779d30 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserManagement.java @@ -6,28 +6,25 @@ import java.util.Collection; import java.util.List; /** - * Management of {@link UserId}s and {@link RoleId}s, used for access control with {@link Role}s. + * Management of {@link UserId}s as members of {@link Role}s. * * @author jonmv */ public interface UserManagement { /** Creates the given role, or throws if the role already exists. */ - void createRole(RoleId role); + void createRole(Role role); - /** Deletes the given role, or throws if it doesn't already exist. */ - void deleteRole(RoleId role); + /** Ensures the given role does not exist. */ + void deleteRole(Role role); /** Ensures the given users exist, and are part of the given role, or throws if the role does not exist. */ - void addUsers(RoleId role, Collection<UserId> users); + void addUsers(Role role, Collection<UserId> users); /** Ensures none of the given users are part of the given role, or throws if the role does not exist. */ - void removeUsers(RoleId role, Collection<UserId> users); - - /** Returns all known roles. */ - List<RoleId> listRoles(); + void removeUsers(Role role, Collection<UserId> users); /** Returns all users in the given role, or throws if the role does not exist. */ - List<UserId> listUsers(RoleId role); + List<UserId> listUsers(Role role); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserRoles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserRoles.java new file mode 100644 index 00000000000..479fcbd2589 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserRoles.java @@ -0,0 +1,123 @@ +package com.yahoo.vespa.hosted.controller.api.integration.user; + +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; +import com.yahoo.vespa.hosted.controller.api.role.Roles; +import com.yahoo.vespa.hosted.controller.api.role.TenantRole; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * Validation, utility and serialization methods for roles used in user management. + * + * @author jonmv + */ +public class UserRoles { + + private final Roles roles; + + /** Creates a new UserRoles which can be used for serialisation and listing of bound user roles. */ + public UserRoles(Roles roles) { + this.roles = requireNonNull(roles); + } + + /** Returns the list of {@link TenantRole}s a {@link UserId} may be a member of. */ + public List<TenantRole> tenantRoles(TenantName tenant) { + return List.of(roles.tenantOwner(tenant), + roles.tenantAdmin(tenant), + roles.tenantOperator(tenant)); + } + + /** Returns the list of {@link ApplicationRole}s a {@link UserId} may be a member of. */ + public List<ApplicationRole> applicationRoles(TenantName tenant, ApplicationName application) { + return List.of(roles.applicationAdmin(tenant, application), + roles.applicationOperator(tenant, application), + roles.applicationDeveloper(tenant, application), + roles.applicationReader(tenant, application)); + } + + /** Returns the {@link Role} the given value represents. */ + public Role toRole(String value) { + String[] parts = value.split("\\."); + if (parts.length == 1) return toOperatorRole(parts[0]); + if (parts.length == 2) return toRole(TenantName.from(parts[0]), parts[1]); + if (parts.length == 3) return toRole(TenantName.from(parts[0]), ApplicationName.from(parts[1]), parts[2]); + throw new IllegalArgumentException("Malformed or illegal role value '" + value + "'."); + } + + public Role toOperatorRole(String roleName) { + switch (roleName) { + case "hostedOperator": return roles.hostedOperator(); + default: throw new IllegalArgumentException("Malformed or illegal role name '" + roleName + "'."); + } + } + + /** Returns the {@link Role} the given tenant, application and role names correspond to. */ + public Role toRole(TenantName tenant, String roleName) { + switch (roleName) { + case "tenantOwner": return roles.tenantOwner(tenant); + case "tenantAdmin": return roles.tenantAdmin(tenant); + case "tenantOperator": return roles.tenantOperator(tenant); + default: throw new IllegalArgumentException("Malformed or illegal role name '" + roleName + "'."); + } + } + + /** Returns the {@link Role} the given tenant and role names correspond to. */ + public Role toRole(TenantName tenant, ApplicationName application, String roleName) { + switch (roleName) { + case "applicationAdmin": return roles.applicationAdmin(tenant, application); + case "applicationOperator": return roles.applicationOperator(tenant, application); + case "applicationDeveloper": return roles.applicationDeveloper(tenant, application); + case "applicationReader": return roles.applicationReader(tenant, application); + default: throw new IllegalArgumentException("Malformed or illegal role name '" + roleName + "'."); + } + } + + /** Returns a serialised representation the given role. */ + public static String valueOf(Role role) { + if (role instanceof TenantRole) return valueOf((TenantRole) role); + if (role instanceof ApplicationRole) return valueOf((ApplicationRole) role); + throw new IllegalArgumentException("Unexpected role type '" + role.getClass().getName() + "'."); + } + + private static String valueOf(TenantRole role) { + return valueOf(role.tenant()) + "." + valueOf(role.definition()); + } + + private static String valueOf(ApplicationRole role) { + return valueOf(role.tenant()) + "." + valueOf(role.application()) + "." + valueOf(role.definition()); + } + + private static String valueOf(TenantName tenant) { + if (tenant.value().contains(".")) + throw new IllegalArgumentException("Tenant names may not contain '.'."); + + return tenant.value(); + } + + private static String valueOf(ApplicationName application) { + if (application.value().contains(".")) + throw new IllegalArgumentException("Application names may not contain '.'."); + + return application.value(); + } + + private static String valueOf(RoleDefinition role) { + switch (role) { + case tenantOwner: return "tenantOwner"; + case tenantAdmin: return "tenantAdmin"; + case tenantOperator: return "tenantOperator"; + case applicationAdmin: return "applicationAdmin"; + case applicationOperator: return "applicationOperator"; + case applicationDeveloper: return "applicationDeveloper"; + case applicationReader: return "applicationReader"; + default: throw new IllegalArgumentException("No value defined for role '" + role + "'."); + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index d085d00baaa..db9291cd651 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -6,6 +6,10 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneFilter; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -50,7 +54,7 @@ public interface ZoneRegistry { SystemName system(); /** Return the configserver's Athenz service identity */ - AthenzService getConfigServerAthenzService(ZoneId zoneId); + AthenzIdentity getConfigServerAthenzIdentity(ZoneId zoneId); /** Returns the Vespa upgrade policy to use for zones in this registry */ UpgradePolicy upgradePolicy(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java index 3ba0367a00c..14d8d06d0c6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java @@ -9,7 +9,7 @@ import java.util.Objects; import java.util.Optional; /** - * The context in which a role is valid. + * The context in which a role is valid. This is immutable. * * @author mpolden */ @@ -40,11 +40,6 @@ public class Context { return system; } - /** Returns whether this context is considered limited */ - public boolean limited() { - return tenant.isPresent() || application.isPresent(); - } - /** Returns a context that has no restrictions on tenant or application in given system */ public static Context unlimitedIn(SystemName system) { return new Context(Optional.empty(), Optional.empty(), system); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index edf3f4e8711..23bf8514b9c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -15,6 +15,7 @@ import java.util.Set; * When creating a new API, its paths must be added here and a policy must be declared in {@link Policy}. * * @author mpolden + * @author jonmv */ public enum PathGroup { @@ -27,30 +28,43 @@ public enum PathGroup { "/provision/v2/{*}", "/zone/v2/{*}"), - /** Paths used for user management. */ - userManagement("/user/v1/{*}"), // TODO probably add tenant and application levels. - /** Paths used for creating user tenants. */ user("/application/v4/user"), /** Paths used for creating tenants with proper access control. */ tenant(Matcher.tenant, + Optional.of("/api"), "/application/v4/tenant/{tenant}"), + /** Paths used for user management on the tenant level. */ + tenantUsers(Matcher.tenant, + Optional.of("/api"), + "/user/v1/tenant/{tenant}"), + /** Paths used by tenant administrators. */ tenantInfo(Matcher.tenant, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/"), /** Path for the base application resource. */ application(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}"), + /** Paths used for user management on the application level. */ + applicationUsers(Matcher.tenant, + Matcher.application, + Optional.of("/api"), + "/user/v1/tenant/{tenant}/application/{application}"), + /** Paths used by application administrators. */ applicationInfo(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/deploying/{*}", "/application/v4/tenant/{tenant}/application/{application}/instance/{*}", + "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{*}", @@ -59,10 +73,12 @@ public enum PathGroup { /** Path used to restart application nodes. */ // TODO move to the above when everyone is on new pipeline. applicationRestart(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/restart"), /** Paths used for development deployments. */ developmentDeployment(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{instance}", @@ -71,6 +87,7 @@ public enum PathGroup { /** Paths used for production deployments. */ productionDeployment(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{instance}", @@ -81,55 +98,72 @@ public enum PathGroup { /** Paths used for continuous deployment to production. */ submission(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/submit"), /** Paths used for other tasks by build services. */ // TODO: This will vanish. buildService(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", "/application/v4/tenant/{tenant}/application/{application}/promote", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), + /** Paths which contain (not very strictly) classified information about customers. */ + classifiedTenantInfo(Optional.of("/api"), + "/application/v4/", + "/application/v4/tenant/"), + /** Paths which contain (not very strictly) classified information about, e.g., customers. */ classifiedInfo("/athenz/v1/{*}", "/cost/v1/{*}", "/deployment/v1/{*}", - "/application/v4/", - "/application/v4/tenant/", "/", "/d/{*}", - "/statuspage/v1/{*}" - ), + "/statuspage/v1/{*}"), /** Paths providing public information. */ publicInfo("/badge/v1/{*}", "/zone/v1/{*}"); final List<String> pathSpecs; + final String prefix; final List<Matcher> matchers; PathGroup(String... pathSpecs) { - this(List.of(), List.of(pathSpecs)); + this(List.of(), Optional.empty(), List.of(pathSpecs)); + } + + PathGroup(Optional<String> prefix, String... pathSpecs) { + this(List.of(), prefix, List.of(pathSpecs)); } PathGroup(Matcher first, String... pathSpecs) { - this(List.of(first), List.of(pathSpecs)); + this(List.of(first), Optional.empty(), List.of(pathSpecs)); + } + + PathGroup(Matcher first, Optional<String> prefix, String... pathSpecs) { + this(List.of(first), prefix, List.of(pathSpecs)); } PathGroup(Matcher first, Matcher second, String... pathSpecs) { - this(List.of(first, second), List.of(pathSpecs)); + this(List.of(first, second), Optional.empty(), List.of(pathSpecs)); + } + + PathGroup(Matcher first, Matcher second, Optional<String> prefix, String... pathSpecs) { + this(List.of(first, second), prefix, List.of(pathSpecs)); } /** Creates a new path group, if the given context matchers are each present exactly once in each of the given specs. */ - PathGroup(List<Matcher> matchers, List<String> pathSpecs) { + PathGroup(List<Matcher> matchers, Optional<String> prefix, List<String> pathSpecs) { this.matchers = matchers; + this.prefix = prefix.orElse(""); this.pathSpecs = pathSpecs; } /** Returns path if it matches any spec in this group, with match groups set by the match. */ - @SuppressWarnings("deprecation") private Optional<Path> get(URI uri) { - Path matcher = new Path(uri); // TODO Get URI down here. + Path matcher = new Path(uri, prefix); for (String spec : pathSpecs) // Iterate to be sure the Path's state is that of the match. if (matcher.matches(spec)) return Optional.of(matcher); return Optional.empty(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 970717b14a3..db4cca20b9a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -23,10 +23,15 @@ public enum Policy { .on(PathGroup.all()) .in(SystemName.all())), - /** Full access to user management in select systems. */ - manager(Privilege.grant(Action.all()) - .on(PathGroup.userManagement) - .in(SystemName.Public)), + /** Full access to user management for a tenant in select systems. */ + tenantManager(Privilege.grant(Action.all()) + .on(PathGroup.tenantUsers) + .in(SystemName.all())), + + /** Full access to user management for an application in select systems. */ + applicationManager(Privilege.grant(Action.all()) + .on(PathGroup.applicationUsers) + .in(SystemName.all())), /** Access to create a user tenant in select systems. */ userCreate(Privilege.grant(Action.update) @@ -50,7 +55,7 @@ public enum Policy { /** Read access to tenant information and settings. */ tenantRead(Privilege.grant(Action.read) - .on(PathGroup.tenant, PathGroup.tenantInfo) + .on(PathGroup.tenant, PathGroup.tenantInfo, PathGroup.tenantUsers, PathGroup.applicationUsers) .in(SystemName.all())), /** Access to create application under a certain tenant. */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index 86d59b4bbb6..c28fa7a3fc3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.api.role; import java.net.URI; import java.util.Objects; -import static java.util.Objects.requireNonNull; - /** * A role is a combination of a {@link RoleDefinition} and a {@link Context}, which allows evaluation * of access control for a given action on a resource. Create using {@link Roles}. @@ -18,18 +16,28 @@ public abstract class Role { final Context context; Role(RoleDefinition roleDefinition, Context context) { - this.roleDefinition = requireNonNull(roleDefinition); - this.context = requireNonNull(context); + this.roleDefinition = Objects.requireNonNull(roleDefinition); + this.context = Objects.requireNonNull(context); } /** Returns the role definition of this bound role. */ public RoleDefinition definition() { return roleDefinition; } /** Returns whether this role is allowed to perform the given action on the given resource. */ - public boolean allows(Action action, URI uri) { + public final boolean allows(Action action, URI uri) { return roleDefinition.policies().stream().anyMatch(policy -> policy.evaluate(action, uri, context)); } + /** Returns whether the other role is a parent of this, and has a context included in this role's context. */ + public boolean implies(Role other) { + if ( ! context.system().equals(other.context.system())) + throw new IllegalStateException("Coexisting roles should always be in the same system."); + + return (context.tenant().isEmpty() || context.tenant().equals(other.context.tenant())) + && (context.application().isEmpty() || context.application().equals(other.context.application())) + && roleDefinition.inherited().contains(other.roleDefinition); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index e9c2f7bc643..af068decc83 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.controller.api.role; import java.util.EnumSet; +import java.util.HashSet; import java.util.Set; /** @@ -40,27 +41,28 @@ public enum RoleDefinition { Policy.developmentDeployment), /** Application operator with access to normal, operational tasks of an application. */ - applicationOperator(applicationDeveloper, + applicationOperator(applicationReader, Policy.applicationOperations), /** Application administrator with full access to an already existing application, including emergency operations. */ - applicationAdmin(applicationOperator, + applicationAdmin(applicationDeveloper, + applicationOperator, Policy.applicationUpdate, + Policy.applicationDelete, + Policy.applicationManager, Policy.productionDeployment, Policy.submission), - /** Application administrator with the additional ability to delete an application. */ - applicationOwner(applicationOperator, - Policy.applicationDelete), - - /** Tenant operator with admin access to all applications under the tenant, as well as the ability to create applications. */ - tenantOperator(applicationAdmin, + /** Tenant operator with access to create application under a tenant, and to read the tenant's and public data. */ + tenantOperator(everyone, + Policy.tenantRead, Policy.applicationCreate), /** Tenant admin with full access to all tenant resources, except deleting the tenant. */ tenantAdmin(tenantOperator, + applicationAdmin, Policy.applicationDelete, - Policy.manager, + Policy.tenantManager, Policy.tenantUpdate), /** Tenant admin with full access to all tenant resources. */ @@ -84,17 +86,36 @@ public enum RoleDefinition { Policy.applicationOperations, Policy.developmentDeployment); + private final Set<RoleDefinition> parents; private final Set<Policy> policies; RoleDefinition(Policy... policies) { - this.policies = EnumSet.copyOf(Set.of(policies)); + this(Set.of(), policies); } - RoleDefinition(RoleDefinition inherited, Policy... policies) { + RoleDefinition(RoleDefinition first, Policy... policies) { + this(Set.of(first), policies); + } + + RoleDefinition(RoleDefinition first, RoleDefinition second, Policy... policies) { + this(Set.of(first, second), policies); + } + + RoleDefinition(Set<RoleDefinition> parents, Policy... policies) { + this.parents = new HashSet<>(parents); this.policies = EnumSet.copyOf(Set.of(policies)); - this.policies.addAll(inherited.policies); + for (RoleDefinition parent : parents) { + this.parents.addAll(parent.parents); + this.policies.addAll(parent.policies); + } + } + + Set<Policy> policies() { + return policies; } - Set<Policy> policies() { return policies; } + Set<RoleDefinition> inherited() { + return parents; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Roles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Roles.java index f6149bf6e88..bfdb0be4378 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Roles.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Roles.java @@ -19,7 +19,6 @@ public class Roles { private final SystemName system; - @Inject public Roles(ZoneRegistry zones) { this(zones.system()); @@ -71,11 +70,6 @@ public class Roles { return new TenantRole(RoleDefinition.tenantOperator, system, tenant); } - /** Returns a {@link RoleDefinition#applicationOwner} for the current system and given tenant and application. */ - public ApplicationRole applicationOwner(TenantName tenant, ApplicationName application) { - return new ApplicationRole(RoleDefinition.applicationOwner, system, tenant, application); - } - /** Returns a {@link RoleDefinition#applicationAdmin} for the current system and given tenant and application. */ public ApplicationRole applicationAdmin(TenantName tenant, ApplicationName application) { return new ApplicationRole(RoleDefinition.applicationAdmin, system, tenant, application); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java index 41444258a68..3378f9e0061 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java @@ -5,8 +5,9 @@ import java.security.Principal; import java.util.Objects; import java.util.Set; -import static java.util.Objects.requireNonNull; - +/** + * @author tokle + */ public class SecurityContext { public static final String ATTRIBUTE_NAME = SecurityContext.class.getName(); @@ -15,7 +16,7 @@ public class SecurityContext { private final Set<Role> roles; public SecurityContext(Principal principal, Set<Role> roles) { - this.principal = requireNonNull(principal); + this.principal = Objects.requireNonNull(principal); this.roles = Set.copyOf(roles); } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java index a01e7c22d35..8e278240a02 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.identifiers; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleIdTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleIdTest.java deleted file mode 100644 index 609646eb672..00000000000 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleIdTest.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.yahoo.vespa.hosted.controller.api.integration.user; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; -import com.yahoo.vespa.hosted.controller.api.role.Roles; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author jonmv - */ -public class RoleIdTest { - - @Test - public void testSerialization() { - Roles roles = new Roles(SystemName.main); - - TenantName tenant = TenantName.from("my-tenant"); - for (TenantRole role : List.of(roles.tenantOwner(tenant), - roles.tenantAdmin(tenant), - roles.tenantOperator(tenant))) - assertEquals(role, RoleId.fromRole(role).toRole(roles)); - - ApplicationName application = ApplicationName.from("my-application"); - for (ApplicationRole role : List.of(roles.applicationOwner(tenant, application), - roles.applicationAdmin(tenant, application), - roles.applicationOperator(tenant, application), - roles.applicationDeveloper(tenant, application), - roles.applicationReader(tenant, application))) - assertEquals(role, RoleId.fromRole(role).toRole(roles)); - - assertEquals(roles.tenantOperator(tenant), - RoleId.fromValue("my-tenant.tenantOperator").toRole(roles)); - assertEquals(roles.applicationReader(tenant, application), - RoleId.fromValue("my-tenant.my-application.applicationReader").toRole(roles)); - } - - @Test(expected = IllegalArgumentException.class) - public void illegalTenantName() { - RoleId.fromRole(new Roles(SystemName.main).tenantAdmin(TenantName.from("my.tenant"))); - } - - @Test(expected = IllegalArgumentException.class) - public void illegalApplicationName() { - RoleId.fromRole(new Roles(SystemName.main).applicationOperator(TenantName.from("my-tenant"), ApplicationName.from("my.app"))); - } - - @Test(expected = IllegalArgumentException.class) - public void illegalRole() { - RoleId.fromRole(new Roles(SystemName.main).tenantPipeline(TenantName.from("my-tenant"), ApplicationName.from("my-app"))); - } - - @Test(expected = IllegalArgumentException.class) - public void illegalRoleValue() { - RoleId.fromValue("my-tenant.awesomePerson").toRole(new Roles(SystemName.cd)); - } - - @Test(expected = IllegalArgumentException.class) - public void illegalCombination() { - RoleId.fromValue("my-tenant.my-application.tenantOwner").toRole(new Roles(SystemName.cd)); - } - - @Test(expected = IllegalArgumentException.class) - public void illegalValue() { - RoleId.fromValue("hostedOperator").toRole(new Roles(SystemName.Public)); - } - -} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserRolesTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserRolesTest.java new file mode 100644 index 00000000000..89df7a24559 --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/UserRolesTest.java @@ -0,0 +1,72 @@ +package com.yahoo.vespa.hosted.controller.api.integration.user; + +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; +import com.yahoo.vespa.hosted.controller.api.role.Roles; +import com.yahoo.vespa.hosted.controller.api.role.TenantRole; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class UserRolesTest { + + private static final Roles roles = new Roles(SystemName.main); + private static final UserRoles userRoles = new UserRoles(roles); + + @Test + public void testSerialization() { + TenantName tenant = TenantName.from("my-tenant"); + for (TenantRole role : userRoles.tenantRoles(tenant)) + assertEquals(role, userRoles.toRole(UserRoles.valueOf(role))); + + ApplicationName application = ApplicationName.from("my-application"); + for (ApplicationRole role : userRoles.applicationRoles(tenant, application)) + assertEquals(role, userRoles.toRole(UserRoles.valueOf(role))); + + assertEquals(roles.tenantOperator(tenant), + userRoles.toRole("my-tenant.tenantOperator")); + assertEquals(roles.applicationReader(tenant, application), + userRoles.toRole("my-tenant.my-application.applicationReader")); + } + + @Test(expected = IllegalArgumentException.class) + public void illegalTenantName() { + UserRoles.valueOf(roles.tenantAdmin(TenantName.from("my.tenant"))); + } + + @Test(expected = IllegalArgumentException.class) + public void illegalApplicationName() { + UserRoles.valueOf(roles.applicationOperator(TenantName.from("my-tenant"), ApplicationName.from("my.app"))); + } + + @Test(expected = IllegalArgumentException.class) + public void illegalRole() { + UserRoles.valueOf(roles.tenantPipeline(TenantName.from("my-tenant"), ApplicationName.from("my-app"))); + } + + @Test(expected = IllegalArgumentException.class) + public void illegalRoleValue() { + userRoles.toRole("my-tenant.awesomePerson"); + } + + @Test(expected = IllegalArgumentException.class) + public void illegalCombination() { + userRoles.toRole("my-tenant.my-application.tenantOwner"); + } + + @Test(expected = IllegalArgumentException.class) + public void illegalValue() { + userRoles.toRole("everyone"); + } + + @Test + public void allowHostedOperator() { + assertEquals(roles.hostedOperator(), userRoles.toRole("hostedOperator")); + } + +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 1badd157b1b..6cfe01cfb77 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -51,4 +51,53 @@ public class RoleTest { assertFalse("No global read access", role.allows(Action.read, URI.create("/controller/v1/foo"))); } + @Test + public void implications() { + Roles roles = new Roles(SystemName.main); + TenantName tenant1 = TenantName.from("t1"); + ApplicationName application1 = ApplicationName.from("a1"); + TenantName tenant2 = TenantName.from("t2"); + ApplicationName application2 = ApplicationName.from("a2"); + + Role tenantOwner1 = roles.tenantOwner(tenant1); + Role tenantAdmin1 = roles.tenantAdmin(tenant1); + Role tenantAdmin2 = roles.tenantAdmin(tenant2); + Role tenantOperator1 = roles.tenantOperator(tenant1); + Role applicationAdmin11 = roles.applicationAdmin(tenant1, application1); + Role applicationOperator11 = roles.applicationOperator(tenant1, application1); + Role applicationDeveloper11 = roles.applicationDeveloper(tenant1, application1); + Role applicationReader11 = roles.applicationReader(tenant1, application1); + Role applicationReader12 = roles.applicationReader(tenant1, application2); + Role applicationReader22 = roles.applicationReader(tenant2, application2); + + assertFalse(tenantOwner1.implies(tenantOwner1)); + assertTrue(tenantOwner1.implies(tenantAdmin1)); + assertFalse(tenantOwner1.implies(tenantAdmin2)); + assertTrue(tenantOwner1.implies(tenantOperator1)); + assertTrue(tenantOwner1.implies(applicationAdmin11)); + assertTrue(tenantOwner1.implies(applicationReader11)); + assertTrue(tenantOwner1.implies(applicationReader12)); + assertFalse(tenantOwner1.implies(applicationReader22)); + + assertFalse(tenantAdmin1.implies(tenantOwner1)); + assertFalse(tenantAdmin1.implies(tenantAdmin2)); + assertTrue(tenantAdmin1.implies(applicationDeveloper11)); + + assertFalse(tenantOperator1.implies(applicationReader11)); + + assertFalse(applicationAdmin11.implies(tenantAdmin1)); + assertFalse(applicationAdmin11.implies(tenantOperator1)); + assertTrue(applicationAdmin11.implies(applicationOperator11)); + assertTrue(applicationAdmin11.implies(applicationDeveloper11)); + assertTrue(applicationAdmin11.implies(applicationReader11)); + assertFalse(applicationAdmin11.implies(applicationReader12)); + assertFalse(applicationAdmin11.implies(applicationReader22)); + + assertFalse(applicationOperator11.implies(applicationDeveloper11)); + assertTrue(applicationOperator11.implies(applicationReader11)); + + assertFalse(applicationDeveloper11.implies(applicationOperator11)); + assertTrue(applicationDeveloper11.implies(applicationReader11)); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index ad30adee576..84e15deea4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -9,16 +9,16 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -199,9 +199,10 @@ public class Application { return rotation; } - /** Returns the global rotation dns name, if present */ - public Optional<GlobalDnsName> globalDnsName(SystemName system) { - return rotation.map(ignored -> new GlobalDnsName(id, system)); + /** Returns the default global endpoints for this in given system */ + public EndpointList endpointsIn(SystemName system) { + if (rotation.isEmpty()) return EndpointList.EMPTY; + return EndpointList.defaultGlobal(id, system); } public Optional<String> pemDeployKey() { return pemDeployKey; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index b6993fbc421..4c5be570a02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -41,11 +41,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; @@ -283,7 +284,7 @@ public class ApplicationController { ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set<String> rotationNames = new HashSet<>(); - Set<String> cnames = new HashSet<>(); + Set<String> cnames; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -324,13 +325,7 @@ public class ApplicationController { // Assign global rotation application = withRotation(application, zone); Application app = application.get(); - app.globalDnsName(controller.system()).ifPresent(applicationRotation -> { - rotationNames.add(app.rotation().orElseThrow(() -> new RuntimeException("Global Dns assigned, but no rotation id present")).asString()); - cnames.add(applicationRotation.dnsName()); - cnames.add(applicationRotation.secureDnsName()); - cnames.add(applicationRotation.oathDnsName()); - }); - + cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); // Update application with information from application package if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally() @@ -438,18 +433,20 @@ public class ApplicationController { application = application.with(rotation.id()); store(application); // store assigned rotation even if deployment fails - GlobalDnsName dnsName = application.get().globalDnsName(controller.system()) - .orElseThrow(() -> new IllegalStateException("Expected rotation to be assigned")); boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) .value(); - registerCname(dnsName.oathDnsName(), rotation.name()); - if (redirectLegacyDns) { - registerCname(dnsName.dnsName(), dnsName.oathDnsName()); - registerCname(dnsName.secureDnsName(), dnsName.oathDnsName()); - } else { - registerCname(dnsName.dnsName(), rotation.name()); - registerCname(dnsName.secureDnsName(), rotation.name()); - } + + EndpointList globalEndpoints = application.get() + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + globalEndpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + if (redirectLegacyDns) { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); + } else { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); + } + }); } } return application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 7754286ba9e..a7596ce808f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; import com.yahoo.vespa.hosted.controller.deployment.JobController; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 0cfbc60ad8c..c82c0d1be06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 51ac25bc321..b9be68f6de1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -5,7 +5,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.time.Instant; import java.util.Collections; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java new file mode 100644 index 00000000000..14a5d3c7ddf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -0,0 +1,281 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.net.URI; +import java.util.Objects; + +/** + * Represents an application's endpoint. The endpoint scope can either be global or a specific zone. This is visible to + * the tenant and is used by the tenant when accessing deployments. + * + * @author mpolden + */ +public class Endpoint { + + public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; + public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; + + private final URI url; + private final Scope scope; + private final boolean legacy; + private final boolean directRouting; + + private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy, + boolean directRouting) { + Objects.requireNonNull(name, "name must be non-null"); + Objects.requireNonNull(application, "application must be non-null"); + Objects.requireNonNull(system, "system must be non-null"); + Objects.requireNonNull(port, "port must be non-null"); + this.url = createUrl(name, application, zone, system, port, legacy, directRouting); + this.scope = zone == null ? Scope.global : Scope.zone; + this.legacy = legacy; + this.directRouting = directRouting; + } + + /** Returns the URL used to access this */ + public URI url() { + return url; + } + + /** Returns the DNS name of this */ + public String dnsName() { + return url.getHost(); + } + + /** Returns the scope of this */ + public Scope scope() { + return scope; + } + + /** Returns whether this is considered a legacy DNS name that is due for removal */ + public boolean legacy() { + return legacy; + } + + /** + * Returns whether this endpoint supports direct routing. Direct routing means that this endpoint is served by an + * exclusive load balancer instead of a shared routing layer. + */ + public boolean directRouting() { + return directRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Endpoint endpoint = (Endpoint) o; + return url.equals(endpoint.url); + } + + @Override + public int hashCode() { + return Objects.hash(url); + } + + @Override + public String toString() { + return String.format("endpoint %s [scope=%s, legacy=%s, directRouting=%s]", url, scope, legacy, directRouting); + } + + private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system, + Port port, boolean legacy, boolean directRouting) { + String scheme = port.tls ? "https" : "http"; + String separator = separator(system, directRouting, port.tls); + String portPart = port.isDefault() ? "" : ":" + port.port; + return URI.create(scheme + "://" + + sanitize(namePart(name, separator)) + + systemPart(system, separator) + + sanitize(instancePart(application, zone, separator)) + + sanitize(application.application().value()) + + separator + + sanitize(application.tenant().value()) + + "." + + scopePart(zone, legacy) + + dnsSuffix(system, legacy) + + portPart + + "/"); + } + + private static String sanitize(String part) { // TODO: Reject reserved words + return part.replace('_', '-'); + } + + private static String separator(SystemName system, boolean directRouting, boolean tls) { + if (!tls) return "."; + if (directRouting) return "."; + if (isPublic(system)) return "."; + return "--"; + } + + private static String namePart(String name, String separator) { + if ("default".equals(name)) return ""; + return name + separator; + } + + private static String scopePart(ZoneId zone, boolean legacy) { + if (zone == null) return "global"; + if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints + return zone.region().value() + "." + zone.environment().value(); + } + + private static String instancePart(ApplicationId application, ZoneId zone, String separator) { + if (zone == null) return ""; // Always omit instance for global endpoints + if (application.instance().isDefault()) return ""; // Skip "default" + return application.instance().value() + separator; + } + + private static String systemPart(SystemName system, String separator) { + if (system == SystemName.main || isPublic(system)) return ""; + return system.name() + separator; + } + + private static String dnsSuffix(SystemName system, boolean legacy) { + switch (system) { + case cd: + case main: + if (legacy) return YAHOO_DNS_SUFFIX; + return OATH_DNS_SUFFIX; + case Public: + case vaas: + return PUBLIC_DNS_SUFFIX; + default: throw new IllegalArgumentException("No DNS suffix declared for system " + system); + } + } + + private static boolean isPublic(SystemName system) { // TODO: Remove and inline once we're down to one + return system == SystemName.Public || system == SystemName.vaas; + } + + /** An endpoint's scope */ + public enum Scope { + + /** Endpoint points to all zones */ + global, + + /** Endpoint points to a single zone */ + zone, + + } + + /** Represents an endpoint's HTTP port */ + public static class Port { + + private final int port; + private final boolean tls; + + private Port(int port, boolean tls) { + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); + } + this.port = port; + this.tls = tls; + } + + private boolean isDefault() { + return port == 80 || port == 443; + } + + /** Returns the default HTTPS port */ + public static Port tls() { + return new Port(443, true); + } + + /** Create a HTTPS port */ + public static Port tls(int port) { + return new Port(port, true); + } + + /** Create a HTTP port */ + public static Port plain(int port) { + return new Port(port, false); + } + + } + + /** Build an endpoint for given application */ + public static EndpointBuilder of(ApplicationId application) { + return new EndpointBuilder(application); + } + + public static class EndpointBuilder { + + private final ApplicationId application; + + private ZoneId zone; + private ClusterSpec.Id cluster; + private RotationName rotation; + private Port port; + private boolean legacy = false; + private boolean directRouting = false; + + private EndpointBuilder(ApplicationId application) { + this.application = application; + } + + /** Sets the cluster and zone target of this */ + public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { + if (rotation != null) { + throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + } + this.cluster = cluster; + this.zone = zone; + return this; + } + + /** Sets the rotation target of this */ + public EndpointBuilder target(RotationName rotation) { + if (cluster != null && zone != null) { + throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + } + this.rotation = rotation; + return this; + } + + /** Sets the port of this */ + public EndpointBuilder on(Port port) { + this.port = port; + return this; + } + + /** Marks this as a legacy endpoint */ + public EndpointBuilder legacy() { + this.legacy = true; + return this; + } + + /** Enables direct routing support for this */ + public EndpointBuilder directRouting() { + this.directRouting = true; + return this; + } + + /** Sets the system that owns this */ + public Endpoint in(SystemName system) { + String name; + if (cluster != null && zone != null) { + name = cluster.value(); + } else if (rotation != null) { + name = rotation.value(); + } else { + throw new IllegalArgumentException("Must set either cluster or rotation target"); + } + if (isPublic(system) && !directRouting) { + throw new IllegalArgumentException("Public system only supports direct routing endpoints"); + } + if (directRouting && !port.isDefault()) { + throw new IllegalArgumentException("Direct routing endpoints only support default port"); + } + return new Endpoint(name, application, zone, system, port, legacy, directRouting); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java new file mode 100644 index 00000000000..0c04a1f099c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -0,0 +1,85 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * A list of endpoints for an application. + * + * @author mpolden + */ +public class EndpointList { + + public static final EndpointList EMPTY = new EndpointList(List.of()); + + private final List<Endpoint> endpoints; + + private EndpointList(List<Endpoint> endpoints) { + long mainEndpoints = endpoints.stream() + .filter(endpoint -> endpoint.scope() == Endpoint.Scope.global) + .filter(Predicate.not(Endpoint::directRouting)) + .filter(Predicate.not(Endpoint::legacy)).count(); + if (mainEndpoints > 1) { + throw new IllegalArgumentException("Can have only 1 non-legacy global endpoint, got " + endpoints); + } + if (endpoints.stream().distinct().count() != endpoints.size()) { + throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); + } + this.endpoints = List.copyOf(endpoints); + } + + public List<Endpoint> asList() { + return endpoints; + } + + /** Returns the main endpoint, if any */ + public Optional<Endpoint> main() { + return endpoints.stream().filter(Predicate.not(Endpoint::legacy)).findFirst(); + } + + /** Returns the subset of endpoints are either legacy or not */ + public EndpointList legacy(boolean legacy) { + return of(endpoints.stream().filter(endpoint -> endpoint.legacy() == legacy)); + } + + /** Returns the subset of endpoints with given scope */ + public EndpointList scope(Endpoint.Scope scope) { + return of(endpoints.stream().filter(endpoint -> endpoint.scope() == scope)); + } + + /** Returns the union of this and given endpoints */ + public EndpointList and(EndpointList endpoints) { + return of(Stream.concat(asList().stream(), endpoints.asList().stream())); + } + + public static EndpointList of(Stream<Endpoint> endpoints) { + return new EndpointList(endpoints.collect(Collectors.toUnmodifiableList())); + } + + /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ + public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { + // Rotation name is always default in the routing layer + RotationName rotation = RotationName.from("default"); + switch (system) { + case cd: + case main: + return new EndpointList(List.of( + Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + )); + } + return EMPTY; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java deleted file mode 100644 index ae638beed5c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.RotationName; -import com.yahoo.config.provision.SystemName; - -import java.net.URI; -import java.util.Objects; - -/** - * Represents names for an application's global rotation. - * - * @author mpolden - */ -public class GlobalDnsName { - - // TODO: TLS: Remove all non-secure stuff when all traffic is on HTTPS. - public static final String DNS_SUFFIX = "global.vespa.yahooapis.com"; - public static final String OATH_DNS_SUFFIX = "global.vespa.oath.cloud"; - private static final int port = 4080; - private static final int securePort = 4443; - - private final URI url; - private final URI secureUrl; - private final URI oathUrl; - - public GlobalDnsName(ApplicationId application, SystemName system) { - this(application, system, null); - } - - public GlobalDnsName(ApplicationId application, SystemName system, RotationName rotation) { - Objects.requireNonNull(application, "application must be non-null"); - Objects.requireNonNull(system, "system must be non-null"); - - this.url = URI.create(String.format("http://%s%s%s.%s.%s:%d/", - clusterPart(rotation, "."), - systemPart(system, "."), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - DNS_SUFFIX, - port)); - this.secureUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/", - clusterPart(rotation, "--"), - systemPart(system, "--"), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - DNS_SUFFIX, - securePort)); - this.oathUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/", - clusterPart(rotation, "--"), - systemPart(system, "--"), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - OATH_DNS_SUFFIX, - securePort)); - } - - /** URL to this rotation */ - public URI url() { - return url; - } - - /** HTTPS URL to this rotation */ - public URI secureUrl() { - return secureUrl; - } - - /** Oath HTTPS URL to this rotation */ - public URI oathUrl() { - return oathUrl; - } - - /** DNS name for this rotation */ - public String dnsName() { - return url.getHost(); - } - - /** DNS name for this rotation */ - public String secureDnsName() { - return secureUrl.getHost(); - } - - /** Oath DNS name for this rotation */ - public String oathDnsName() { - return oathUrl.getHost(); - } - - /** Sanitize by translating '_' to '-' as the former is not allowed in a DNS name */ - private static String sanitize(String s) { - return s.replace('_', '-'); - } - - private static String clusterPart(RotationName rotation, String separator) { - return rotation == null ? "" : rotation.value() + separator; - } - - private static String systemPart(SystemName system, String separator) { - return SystemName.main == system ? "" : system.name() + separator; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java index 33f937f6184..c4b69ce5588 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java @@ -6,14 +6,13 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Represents the DNS routing policy for a load balancer. @@ -41,9 +40,9 @@ public class RoutingPolicy { this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); } - public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, HostName canonicalName, + public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, SystemName system, HostName canonicalName, Optional<String> dnsZone, Set<RotationName> rotations) { - this(owner, zone, HostName.from(aliasOf(cluster, owner, zone)), canonicalName, dnsZone, rotations); + this(owner, zone, HostName.from(endpointOf(cluster, owner, zone, system).dnsName()), canonicalName, dnsZone, rotations); } /** The application owning this */ @@ -76,6 +75,11 @@ public class RoutingPolicy { return rotations; } + /** Endpoints for this routing policy */ + public EndpointList endpointsIn(SystemName system) { + return EndpointList.of(rotations.stream().map(rotation -> endpointOf(owner, rotation, system))); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -98,21 +102,14 @@ public class RoutingPolicy { zone.value()); } - /** Returns the alias to use for the given application cluster in zone */ - private static String aliasOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone) { - List<String> parts = List.of(ignorePartIfDefault(cluster.value()), - ignorePartIfDefault(application.instance().value()), - application.application().value(), - application.tenant().value() + - "." + zone.value() + "." + "vespa.oath.cloud" - ); - return parts.stream() - .filter(Predicate.not(String::isBlank)) - .collect(Collectors.joining("--")); + /** Returns the endpoint of given rotation */ + public static Endpoint endpointOf(ApplicationId application, RotationName rotation, SystemName system) { + return Endpoint.of(application).target(rotation).on(Port.tls()).directRouting().in(system); } - private static String ignorePartIfDefault(String s) { - return "default".equalsIgnoreCase(s) ? "" : s; + /** Returns the endpoint of given cluster */ + public static Endpoint endpointOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone, SystemName system) { + return Endpoint.of(application).target(cluster, zone).on(Port.tls()).directRouting().in(system); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 7a7504830dd..bfd432a0677 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -5,11 +5,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Collections; import java.util.List; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 846c90a96f5..447f9a462b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -28,7 +29,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { } @Override - public AthenzService getControllerIdentity() { + public AthenzIdentity getControllerIdentity() { return identityProvider.identity(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 431020b735b..75b7e137998 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -19,9 +18,9 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; -import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -45,14 +44,14 @@ public class AthenzFacade implements AccessControl { private static final Logger log = Logger.getLogger(AthenzFacade.class.getName()); private final ZmsClient zmsClient; private final ZtsClient ztsClient; - private final AthenzService service; + private final AthenzIdentity service; @Inject public AthenzFacade(AthenzClientFactory factory) { this(factory.createZmsClient(), factory.createZtsClient(), factory.getControllerIdentity()); } - public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzService identity) { + public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzIdentity identity) { this.zmsClient = zmsClient; this.ztsClient = ztsClient; this.service = identity; @@ -142,9 +141,9 @@ public class AthenzFacade implements AccessControl { } @Override - public void createApplication(ApplicationId application, Credentials credentials) { + public void createApplication(ApplicationId id, Credentials credentials) { AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; - createApplication(athenzCredentials.domain(), application.application(), athenzCredentials.token()); + createApplication(athenzCredentials.domain(), id.application(), athenzCredentials.token()); } private void createApplication(AthenzDomain domain, ApplicationName application, OktaAccessToken token) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index f7a8e702b06..37926d944b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -30,23 +29,23 @@ public class ZmsClientMock implements ZmsClient { private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName()); private final AthenzDbMock athenz; - private final AthenzService controllerIdentity; + private final AthenzIdentity controllerIdentity; private static final Pattern TENANT_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.(?<tenantDomain>[\\w\\-_]+)\\..*"); private static final Pattern APPLICATION_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.[\\w\\-_]+\\.res_group\\.(?<resourceGroup>[\\w\\-_]+)\\.wildcard"); - public ZmsClientMock(AthenzDbMock athenz, AthenzService controllerIdentity) { + public ZmsClientMock(AthenzDbMock athenz, AthenzIdentity controllerIdentity) { this.athenz = athenz; this.controllerIdentity = controllerIdentity; } @Override - public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { log("createTenancy(tenantDomain='%s')", tenantDomain); getDomainOrThrow(tenantDomain, false).isVespaTenant = true; } @Override - public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { log("deleteTenancy(tenantDomain='%s')", tenantDomain); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, false); domain.isVespaTenant = false; @@ -55,7 +54,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { log("createProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); ApplicationId applicationId = new ApplicationId(resourceGroup); @@ -65,7 +64,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token) { log("deleteProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); getDomainOrThrow(tenantDomain, true).applications.remove(new ApplicationId(resourceGroup)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java index 8bb5ad12468..5c0407d35a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -45,22 +45,22 @@ public class ZtsClientMock implements ZtsClient { } @Override - public InstanceIdentity registerInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { + public InstanceIdentity registerInstance(AthenzIdentity providerIdentity, AthenzIdentity instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public InstanceIdentity refreshInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { + public InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix) { throw new UnsupportedOperationException(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index 96b7f68e2e0..33db6b95db1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index b58cac27cdb..e6aa8bc51b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.JobState; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 5c9489d415f..23e303a1b62 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -30,7 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index 4a49d5df5ab..c9b07ada854 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index ac080fe15d7..7955505a2b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,11 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 87c0ed8b9ab..e62ed1a9114 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 787a050e59e..0cf89d798a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -65,15 +65,17 @@ public class DeploymentMetricsMaintainer extends Maintainer { .getDeploymentMetrics(application.id(), deployment.zone()); Instant now = controller().clock().instant(); applications.lockIfPresent(application.id(), locked -> { - DeploymentMetrics newMetrics = locked.get().deployments().get(deployment.zone()).metrics() - .withQueriesPerSecond(collectedMetrics.queriesPerSecond()) - .withWritesPerSecond(collectedMetrics.writesPerSecond()) - .withDocumentCount(collectedMetrics.documentCount()) - .withQueryLatencyMillis(collectedMetrics.queryLatencyMillis()) - .withWriteLatencyMillis(collectedMetrics.writeLatencyMillis()) - .at(now); - applications.store(locked.with(deployment.zone(), newMetrics) - .recordActivityAt(now, deployment.zone())); + Deployment existingDeployment = locked.get().deployments().get(deployment.zone()); + if (existingDeployment == null) return; // Deployment removed since we started collecting metrics + DeploymentMetrics newMetrics = existingDeployment.metrics() + .withQueriesPerSecond(collectedMetrics.queriesPerSecond()) + .withWritesPerSecond(collectedMetrics.writesPerSecond()) + .withDocumentCount(collectedMetrics.documentCount()) + .withQueryLatencyMillis(collectedMetrics.queryLatencyMillis()) + .withWriteLatencyMillis(collectedMetrics.writeLatencyMillis()) + .at(now); + applications.store(locked.with(existingDeployment.zone(), newMetrics) + .recordActivityAt(now, existingDeployment.zone())); }); } } catch (Exception e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java index 7693f224b56..c2c68591dea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -84,8 +84,8 @@ public class DnsMaintainer extends Maintainer { /** Returns whether we can update the given record */ private static boolean canUpdate(Record record) { String recordName = record.name().asString(); - return recordName.endsWith(GlobalDnsName.DNS_SUFFIX) || - recordName.endsWith(GlobalDnsName.OATH_DNS_SUFFIX); + return recordName.endsWith(Endpoint.YAHOO_DNS_SUFFIX) || + recordName.endsWith(Endpoint.OATH_DNS_SUFFIX); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index baff67ef4bd..0333711ae39 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -5,8 +5,8 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.yolean.Exceptions; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 5e7db9f58fd..3b521657f15 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -6,7 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.versions.OsVersion; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java index a79eef0244b..417a1944ad3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java @@ -14,8 +14,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.RoutingId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -91,7 +91,8 @@ public class RoutingPolicyMaintainer extends Maintainer { // Create DNS record for each routing ID for (Map.Entry<RoutingId, List<RoutingPolicy>> route : routingTable.entrySet()) { - GlobalDnsName dnsName = dnsName(route.getKey()); + Endpoint endpoint = RoutingPolicy.endpointOf(route.getKey().application(), route.getKey().rotation(), + controller().system()); Set<AliasTarget> targets = route.getValue() .stream() .filter(policy -> policy.dnsZone().isPresent()) @@ -100,10 +101,10 @@ public class RoutingPolicyMaintainer extends Maintainer { policy.zone())) .collect(Collectors.toSet()); try { - nameService.createAlias(RecordName.from(dnsName.oathDnsName()), targets); + nameService.createAlias(RecordName.from(endpoint.dnsName()), targets); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to create or update DNS record for global rotation " + - dnsName.oathDnsName() + ". Retrying in " + maintenanceInterval(), e); + endpoint.dnsName() + ". Retrying in " + maintenanceInterval(), e); } } } @@ -136,7 +137,8 @@ public class RoutingPolicyMaintainer extends Maintainer { /** Register DNS alias for given load balancer */ private RoutingPolicy registerCname(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, loadBalancer.cluster(), + RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, + loadBalancer.cluster(), controller().system(), loadBalancer.hostname(), loadBalancer.dnsZone(), loadBalancer.rotations()); RecordName name = RecordName.from(routingPolicy.alias().value()); @@ -186,23 +188,18 @@ public class RoutingPolicyMaintainer extends Maintainer { Set<RoutingId> activeRoutingIds = routingIdsFrom(loadBalancers); removalCandidates.removeAll(activeRoutingIds); for (RoutingId id : removalCandidates) { - GlobalDnsName dnsName = dnsName(id); + Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller().system()); try { - List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(dnsName.oathDnsName())); + List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName())); nameService.removeRecords(records); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + dnsName.oathDnsName() + + log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + endpoint.dnsName() + "'. Retrying in " + maintenanceInterval()); } } } } - /** Create a global DNS name for given routing ID */ - private GlobalDnsName dnsName(RoutingId routingId) { - return new GlobalDnsName(routingId.application(), controller().system(), routingId.rotation()); - } - /** Compute routing IDs from given load balancers */ private static Set<RoutingId> routingIdsFrom(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { Set<RoutingId> routingIds = new LinkedHashSet<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index ce804f9b7c5..25c9c7f6607 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -5,7 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 8433b2f368c..47ae8566ab8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -20,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index ec34585a950..0bce1872830 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.Run; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 841aedbf5fb..722cde68c65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -8,7 +8,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import java.util.Collections; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 50c70f3e55b..01d9a01a316 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; import org.apache.http.client.config.RequestConfig; @@ -268,7 +268,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { AthenzIdentityVerifier hostnameVerifier = new AthenzIdentityVerifier( singleton( - zoneRegistry.getConfigServerAthenzService( + zoneRegistry.getConfigServerAthenzIdentity( ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); return HttpClientBuilder.create() .setUserAgent("config-server-proxy-client") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8f58827d33a..612c323fc31 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -44,12 +43,13 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; @@ -58,7 +58,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; @@ -132,12 +132,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); switch (request.getMethod()) { - case GET: return handleGET(request); - case PUT: return handlePUT(request); - case POST: return handlePOST(request); - case PATCH: return handlePATCH(request); - case DELETE: return handleDELETE(request); + case GET: return handleGET(path, request); + case PUT: return handlePUT(path, request); + case POST: return handlePOST(path, request); + case PATCH: return handlePATCH(path, request); + case DELETE: return handleDELETE(path, request); case OPTIONS: return handleOPTIONS(); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } @@ -163,8 +164,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handleGET(Path path, HttpRequest request) { if (path.matches("/application/v4/")) return root(request); if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); @@ -173,6 +173,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); @@ -186,8 +187,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handlePUT(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) @@ -195,8 +195,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handlePOST(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); @@ -214,15 +213,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handlePATCH(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handlePATCH(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return setMajorVersion(path.get("tenant"), path.get("application"), request); return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handleDELETE(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); @@ -328,6 +325,58 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .orElseThrow(() -> new NotExistsException(applicationId + " not found")); } + private HttpResponse nodes(String tenantName, String applicationName, String instanceName, String environment, String region) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); + ZoneId zone = ZoneId.from(environment, region); + List<Node> nodes = controller.configServer().nodeRepository().list(zone, id); + + Slime slime = new Slime(); + Cursor nodesArray = slime.setObject().setArray("nodes"); + for (Node node : nodes) { + Cursor nodeObject = nodesArray.addObject(); + nodeObject.setString("hostname", node.hostname().value()); + nodeObject.setString("state", valueOf(node.state())); + nodeObject.setString("orchestration", valueOf(node.serviceState())); + nodeObject.setString("version", node.currentVersion().toString()); + nodeObject.setString("flavor", node.canonicalFlavor()); + nodeObject.setString("clusterId", node.clusterId()); + nodeObject.setString("clusterType", valueOf(node.clusterType())); + } + return new SlimeJsonResponse(slime); + } + + private static String valueOf(Node.State state) { + switch (state) { + case failed: return "failed"; + case parked: return "parked"; + case dirty: return "dirty"; + case ready: return "ready"; + case active: return "active"; + case inactive: return "inactive"; + case reserved: return "reserved"; + case provisioned: return "provisioned"; + default: throw new IllegalArgumentException("Unexpected node state '" + state + "'."); + } + } + + private static String valueOf(Node.ServiceState state) { + switch (state) { + case expectedUp: return "expectedUp"; + case allowedDown: return "allowedDown"; + case unorchestrated: return "unorchestrated"; + default: throw new IllegalArgumentException("Unexpected node state '" + state + "'."); + } + } + + private static String valueOf(Node.ClusterType type) { + switch (type) { + case admin: return "admin"; + case content: return "content"; + case container: return "container"; + default: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'."); + } + } + private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = ZoneId.from(environment, region); @@ -432,21 +481,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Rotation Cursor globalRotationsArray = object.setArray("globalRotations"); + application.endpointsIn(controller.system()) + .scope(Endpoint.Scope.global) + .legacy(false) // Hide legacy names + .asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalRotationsArray::addString); - application.globalDnsName(controller.system()).ifPresent(rotation -> { - globalRotationsArray.addString(rotation.url().toString()); - globalRotationsArray.addString(rotation.secureUrl().toString()); - globalRotationsArray.addString(rotation.oathUrl().toString()); - object.setString("rotationId", application.rotation().get().asString()); - }); + application.rotation().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies(application.id()); for (RoutingPolicy policy : routingPolicies) { - for (RotationName rotation : policy.rotations()) { - GlobalDnsName dnsName = new GlobalDnsName(application.id(), controller.system(), rotation); - globalRotationsArray.addString(dnsName.oathUrl().toString()); - } + policy.endpointsIn(controller.system()).asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalRotationsArray::addString); } // Deployments sorted according to deployment spec @@ -524,6 +575,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); + // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may + // contain the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to + // ask the routing layer here Cursor serviceUrlArray = response.setArray("serviceUrls"); controller.applications().getDeploymentEndpoints(deploymentId) .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 2fb493cad42..48cf2a7824d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index 2429565350c..cfb9bc4f169 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index f25deb11a52..8e397366203 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.Roles; import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; @@ -46,9 +47,9 @@ public class AthenzRoleFilter extends CorsRequestFilterBase { // TODO: No need f private final Roles roles; @Inject - public AthenzRoleFilter(CorsFilterConfig config, AthenzFacade athenz, Controller controller) { + public AthenzRoleFilter(CorsFilterConfig config, AthenzClientFactory athenzClientFactory, Controller controller) { super(Set.copyOf(config.allowedUrls())); - this.athenz = athenz; + this.athenz = new AthenzFacade(athenzClientFactory); this.tenants = controller.tenants(); this.roles = new Roles(controller.system()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 39736d709d0..181731ef896 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -46,7 +46,6 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { @Override public Optional<ErrorResponse> filterRequest(DiscFilterRequest request) { try { - Principal principal = request.getUserPrincipal(); Optional<SecurityContext> securityContext = Optional.ofNullable((SecurityContext)request.getAttribute(SecurityContext.ATTRIBUTE_NAME)); if (securityContext.isEmpty()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 73bbcb1c383..5454d71185a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -14,8 +14,8 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 067e6095b4d..f5532a964fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -1,14 +1,35 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.user; +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.io.IOUtils; import com.yahoo.restapi.Path; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserRoles; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; +import com.yahoo.vespa.hosted.controller.api.role.Roles; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; import com.yahoo.yolean.Exceptions; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -21,19 +42,26 @@ import java.util.logging.Logger; public class UserApiHandler extends LoggingRequestHandler { private final static Logger log = Logger.getLogger(UserApiHandler.class.getName()); + private static final String optionalPrefix = "/api"; - public UserApiHandler(Context parentCtx) { + private final UserRoles roles; + private final UserManagement users; + + @Inject + public UserApiHandler(Context parentCtx, Roles roles, UserManagement users) { super(parentCtx); + this.roles = new UserRoles(roles); + this.users = users; } @Override public HttpResponse handle(HttpRequest request) { try { + Path path = new Path(request.getUri(), optionalPrefix); switch (request.getMethod()) { - case GET: return handleGET(request); - case PUT: return handlePUT(request); - case POST: return handlePOST(request); - case DELETE: return handleDELETE(request); + case GET: return handleGET(path, request); + case POST: return handlePOST(path, request); + case DELETE: return handleDELETE(path, request); case OPTIONS: return handleOPTIONS(); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } @@ -47,33 +75,25 @@ public class UserApiHandler extends LoggingRequestHandler { } } - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri()); - + private HttpResponse handleGET(Path path, HttpRequest request) { + if (path.matches("/user/v1/tenant/{tenant}")) return listTenantRoleMembers(path.get("tenant")); + if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return listApplicationRoleMembers(path.get("tenant"), path.get("application")); return ErrorResponse.notFoundError(String.format("No '%s' handler at '%s'", request.getMethod(), request.getUri().getPath())); } - private HttpResponse handlePUT(HttpRequest request) { - Path path = new Path(request.getUri()); - + private HttpResponse handlePOST(Path path, HttpRequest request) { + if (path.matches("/user/v1/tenant/{tenant}")) return addTenantRoleMember(path.get("tenant"), request); + if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return addApplicationRoleMember(path.get("tenant"), path.get("application"), request); return ErrorResponse.notFoundError(String.format("No '%s' handler at '%s'", request.getMethod(), request.getUri().getPath())); } - private HttpResponse handlePOST(HttpRequest request) { - Path path = new Path(request.getUri()); - - - return ErrorResponse.notFoundError(String.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); - } - - private HttpResponse handleDELETE(HttpRequest request) { - Path path = new Path(request.getUri()); - + private HttpResponse handleDELETE(Path path, HttpRequest request) { + if (path.matches("/user/v1/tenant/{tenant}")) return removeTenantRoleMember(path.get("tenant"), request); + if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return removeApplicationRoleMember(path.get("tenant"), path.get("application"), request); return ErrorResponse.notFoundError(String.format("No '%s' handler at '%s'", request.getMethod(), request.getUri().getPath())); @@ -85,6 +105,108 @@ public class UserApiHandler extends LoggingRequestHandler { return response; } + private HttpResponse listTenantRoleMembers(String tenantName) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("tenant", tenantName); + fillRoles(root, roles.tenantRoles(TenantName.from(tenantName))); + return new SlimeJsonResponse(slime); + } + + private HttpResponse listApplicationRoleMembers(String tenantName, String applicationName) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("tenant", tenantName); + root.setString("application", applicationName); + fillRoles(root, roles.applicationRoles(TenantName.from(tenantName), ApplicationName.from(applicationName))); + return new SlimeJsonResponse(slime); + } + + private void fillRoles(Cursor root, List<? extends Role> roles) { + Cursor rolesArray = root.setArray("roleNames"); + for (Role role : roles) + rolesArray.addString(valueOf(role)); + + Map<UserId, List<Role>> memberships = new HashMap<>(); + for (Role role : roles) + for (UserId user : users.listUsers(role)) { + memberships.putIfAbsent(user, new ArrayList<>()); + memberships.get(user).add(role); + } + Cursor usersArray = root.setArray("users"); + memberships.forEach((user, userRoles) -> { + Cursor userObject = usersArray.addObject(); + userObject.setString("name", user.value()); + Cursor rolesObject = userObject.setObject("roles"); + for (Role role : roles) { + Cursor roleObject = rolesObject.setObject(valueOf(role)); + roleObject.setBool("explicit", userRoles.contains(role)); + roleObject.setBool("implied", userRoles.stream().anyMatch(userRole -> userRole.implies(role))); + } + }); + } + + private HttpResponse addTenantRoleMember(String tenantName, HttpRequest request) { + Inspector requestObject = bodyInspector(request); + String roleName = require("roleName", Inspector::asString, requestObject); + UserId user = new UserId(require("user", Inspector::asString, requestObject)); + Role role = roles.toRole(TenantName.from(tenantName), roleName); + users.addUsers(role, List.of(user)); + return new MessageResponse(user + " is now a member of " + role); + } + + private HttpResponse addApplicationRoleMember(String tenantName, String applicationName, HttpRequest request) { + Inspector requestObject = bodyInspector(request); + String roleName = require("roleName", Inspector::asString, requestObject); + UserId user = new UserId(require("user", Inspector::asString, requestObject)); + Role role = roles.toRole(TenantName.from(tenantName), ApplicationName.from(applicationName), roleName); + users.addUsers(role, List.of(user)); + return new MessageResponse(user + " is now a member of " + role); + } + + private HttpResponse removeTenantRoleMember(String tenantName, HttpRequest request) { + Inspector requestObject = bodyInspector(request); + String roleName = require("roleName", Inspector::asString, requestObject); + UserId user = new UserId(require("user", Inspector::asString, requestObject)); + Role role = roles.toRole(TenantName.from(tenantName), roleName); + if ( role.definition() == RoleDefinition.tenantOwner + && users.listUsers(role).equals(List.of(user))) + throw new IllegalArgumentException("Can't remove the last owner of a tenant."); + + users.removeUsers(role, List.of(user)); + return new MessageResponse(user + " is no longer a member of " + role); + } + + private HttpResponse removeApplicationRoleMember(String tenantName, String applicationName, HttpRequest request) { + Inspector requestObject = bodyInspector(request); + String roleName = require("roleName", Inspector::asString, requestObject); + UserId user = new UserId(require("user", Inspector::asString, requestObject)); + Role role = roles.toRole(TenantName.from(tenantName), ApplicationName.from(applicationName), roleName); + users.removeUsers(role, List.of(user)); + return new MessageResponse(user + " is no longer a member of " + role); + } + + private static Inspector bodyInspector(HttpRequest request) { + return Exceptions.uncheck(() -> SlimeUtils.jsonToSlime(IOUtils.readBytes(request.getData(), 1 << 10)).get()); + } + + private <Type> Type require(String name, Function<Inspector, Type> mapper, Inspector object) { + if ( ! object.field(name).valid()) throw new IllegalArgumentException("Missing field '" + name + "'."); + return mapper.apply(object.field(name)); + } + + private static String valueOf(Role role) { + switch (role.definition()) { + case tenantOwner: return "tenantOwner"; + case tenantAdmin: return "tenantAdmin"; + case tenantOperator: return "tenantOperator"; + case applicationAdmin: return "applicationAdmin"; + case applicationOperator: return "applicationOperator"; + case applicationDeveloper: return "applicationDeveloper"; + case applicationReader: return "applicationReader"; + default: throw new IllegalArgumentException("Unexpected role type '" + role.definition() + "'."); + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 94a2004197a..fcf01f461a1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 65f0abb16c8..9d95383fbfb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -8,8 +8,8 @@ import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java index 4ce797fddb2..77ccce873fe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java @@ -49,10 +49,10 @@ public interface AccessControl { /** * Sets up access control for the given application, based on the given credentials. * - * @param application the ID of the application to create + * @param id the ID of the application to create * @param credentials the credentials for the entity requesting the creation */ - void createApplication(ApplicationId application, Credentials credentials); + void createApplication(ApplicationId id, Credentials credentials); /** * Deletes access control for the given tenant. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index d1806fb5747..008be2fd276 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -2,15 +2,14 @@ package com.yahoo.vespa.hosted.controller.security; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.organization.Marketplace; -import com.yahoo.vespa.hosted.controller.api.integration.user.RoleId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; +import com.yahoo.vespa.hosted.controller.api.integration.user.UserRoles; import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; +import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.Roles; import com.yahoo.vespa.hosted.controller.api.role.TenantRole; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -21,31 +20,30 @@ import java.util.List; /** * @author jonmv - * @author tokle */ public class CloudAccessControl implements AccessControl { private final Marketplace marketplace; private final UserManagement userManagement; private final Roles roles; + private final UserRoles userRoles; @Inject public CloudAccessControl(Marketplace marketplace, UserManagement userManagement, Roles roles) { this.marketplace = marketplace; this.userManagement = userManagement; this.roles = roles; + this.userRoles = new UserRoles(roles); } @Override public CloudTenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { CloudTenantSpec spec = (CloudTenantSpec) tenantSpec; - CloudTenant tenant = new CloudTenant(spec.tenant(), new BillingInfo("customer", "Vespa")); - // CloudTenant tenant new CloudTenant(spec.tenant(), marketplace.resolveCustomer(spec.getRegistrationToken())); - // TODO Enable the above when things work. + CloudTenant tenant = new CloudTenant(spec.tenant(), marketplace.resolveCustomer(spec.getRegistrationToken())); - RoleId ownerRole = RoleId.fromRole(roles.tenantOwner(spec.tenant())); - userManagement.createRole(ownerRole); - userManagement.addUsers(ownerRole, List.of(new UserId(credentials.user().getName()))); + for (Role role : userRoles.tenantRoles(spec.tenant())) + userManagement.createRole(role); + userManagement.addUsers(roles.tenantOwner(spec.tenant()), List.of(new UserId(credentials.user().getName()))); return tenant; } @@ -59,25 +57,21 @@ public class CloudAccessControl implements AccessControl { public void deleteTenant(TenantName tenant, Credentials credentials) { // Probably terminate customer subscription? - tenantRoles(tenant).stream() - .map(RoleId::fromRole) - .filter(userManagement.listRoles()::contains) - .forEach(userManagement::deleteRole); + for (TenantRole role : userRoles.tenantRoles(tenant)) + userManagement.deleteRole(role); } @Override - public void createApplication(ApplicationId application, Credentials credentials) { - RoleId ownerRole = RoleId.fromRole(roles.applicationOwner(application.tenant(), application.application())); - userManagement.createRole(ownerRole); - userManagement.addUsers(ownerRole, List.of(new UserId(credentials.user().getName()))); + public void createApplication(ApplicationId id, Credentials credentials) { + for (Role role : userRoles.applicationRoles(id.tenant(), id.application())) + userManagement.createRole(role); + userManagement.addUsers(roles.applicationAdmin(id.tenant(), id.application()), List.of(new UserId(credentials.user().getName()))); } @Override public void deleteApplication(ApplicationId id, Credentials credentials) { - applicationRoles(id.tenant(), id.application()).stream() - .map(RoleId::fromRole) - .filter(userManagement.listRoles()::contains) - .forEach(userManagement::deleteRole); + for (ApplicationRole role : userRoles.applicationRoles(id.tenant(), id.application())) + userManagement.deleteRole(role); } @Override @@ -87,18 +81,4 @@ public class CloudAccessControl implements AccessControl { return Collections.emptyList(); } - private List<TenantRole> tenantRoles(TenantName tenant) { - return List.of(roles.tenantOperator(tenant), - roles.tenantAdmin(tenant), - roles.tenantOwner(tenant)); - } - - private List<ApplicationRole> applicationRoles(TenantName tenant, ApplicationName application) { - return List.of(roles.applicationReader(tenant, application), - roles.applicationDeveloper(tenant, application), - roles.applicationOperator(tenant, application), - roles.applicationAdmin(tenant, application), - roles.applicationOwner(tenant, application)); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 78d6dababb5..a18c1f47036 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 26410280566..e2d4c90f443 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index bc42b672da4..ada5be83876 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -21,7 +21,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index c18e9c46f07..67033aef52d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -34,7 +34,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java new file mode 100644 index 00000000000..16b875c1892 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -0,0 +1,114 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class EndpointTest { + + private static final ApplicationId app1 = ApplicationId.from("t1", "a1", "default"); + private static final ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); + + @Test + public void test_global_endpoints() { + RotationName rotation = RotationName.from("default"); // Always default for non-direct routing + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint + "http://a1.t1.global.vespa.yahooapis.com:4080/", + Endpoint.of(app1).target(rotation).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Legacy endpoint with TLS + "https://a1--t1.global.vespa.yahooapis.com:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Main endpoint + "https://a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).in(SystemName.main), + + // Main endpoint in CD + "https://cd--a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).in(SystemName.cd), + + // Main endpoint with direct routing and default TLS port + "https://a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).target(rotation).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint with custom rotation name + "https://r1.a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).target(RotationName.from("r1")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance in default rotation + "https://a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).target(rotation).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance with custom rotation name + "https://r2.a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).target(RotationName.from("r2")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint in public system + "https://a1.t1.global.public.vespa.oath.cloud/", + Endpoint.of(app1).target(rotation).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + + @Test + public void test_zone_endpoints() { + ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing + ZoneId prodZone = ZoneId.from("prod", "us-north-1"); + ZoneId testZone = ZoneId.from("test", "us-north-2"); + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint (always contains environment) + "http://a1.t1.us-north-1.prod.vespa.yahooapis.com:4080/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Secure legacy endpoint + "https://a1--t1.us-north-1.prod.vespa.yahooapis.com:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Prod endpoint in main + "https://a1--t1.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + + // Prod endpoint in CD + "https://cd--a1--t1.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.cd), + + // Test endpoint in main + "https://a1--t1.us-north-2.test.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, testZone).on(Port.tls(4443)).in(SystemName.main), + + // Non-default cluster in main + "https://c1--a1--t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).in(SystemName.main), + + // Non-default instance in main + "https://i2--a2--t2.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app2).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + + // Non-default cluster in public + "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/", + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public), + + // Non-default cluster and instance in public + "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/", + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java deleted file mode 100644 index 8d0766829f1..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import org.junit.Test; - -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class RoutingPolicyTest { - - @Test - public void test_endpoint_names() { - ZoneId zoneId = ZoneId.from("prod", "us-north-1"); - ApplicationId withInstance = ApplicationId.from("tenant", "application", "instance"); - testAlias("instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withInstance, zoneId); - testAlias("cluster--instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withInstance, zoneId); - - ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); - testAlias("application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); - testAlias("cluster--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); - } - - private void testAlias(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { - assertEquals(expected, new RoutingPolicy(applicationId, zoneId, ClusterSpec.Id.from(clusterName), - HostName.from("lb-0"), Optional.empty(), Set.of()).alias().value()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index c84f8ed7c58..9b72a60f6ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -6,14 +6,13 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 36261eb05fe..bf7c8cadd3c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index ca8a2e34fdf..096a41e5b3f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index a6755a96fdd..f606d904c5e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import org.junit.Before; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index 04c670cf136..a7b9cbd1e7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -5,7 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import java.util.HashMap; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 803f56fc0d7..927596ca9ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -22,7 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -248,7 +248,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); for (Node node : nodes) { nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(), - node.state(), node.type(), + Node.State.active, + node.type(), node.owner(), node.currentVersion(), application.version().get())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java index d5fa3d14c62..ce2f74e3389 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.HashMap; import java.util.Map; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java index ccd09cb9261..07978e51263 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwne import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Collection; import java.util.List; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index f375d80b28b..49bc910ac33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -8,17 +8,15 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.function.Function; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -91,7 +89,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())) + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())) .forEach(node -> putByHostname(zone, node)); } @@ -133,7 +134,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration() + 1, node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } public void doRestart(DeploymentId deployment, Optional<HostName> hostname) { @@ -149,7 +153,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration() + 1, node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } public void requestReboot(DeploymentId deployment, Optional<HostName> hostname) { @@ -165,7 +172,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration() + 1)); + node.wantedRebootGeneration() + 1, + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } public void doReboot(DeploymentId deployment, Optional<HostName> hostname) { @@ -181,7 +191,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration() + 1, - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 37261b0fdc4..4248a513950 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; @@ -12,10 +13,10 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilter; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneFilterMock; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneFilter; +import com.yahoo.config.provision.zone.ZoneFilterMock; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; @@ -35,12 +36,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>(); private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); private List<ZoneId> zones = new ArrayList<>(); - private SystemName system = SystemName.main; + private SystemName system; private UpgradePolicy upgradePolicy = null; private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); @Inject + public ZoneRegistryMock(ConfigserverConfig config) { + this(SystemName.valueOf(config.system())); + } + public ZoneRegistryMock() { + this(SystemName.main); + } + + public ZoneRegistryMock(SystemName system) { + this.system = system; zones.add(ZoneId.from("prod", "us-east-3")); zones.add(ZoneId.from("prod", "us-west-1")); zones.add(ZoneId.from("prod", "us-central-1")); @@ -91,7 +101,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return ZoneFilterMock.from(Collections.unmodifiableList(zones)); } - public AthenzService getConfigServerAthenzService(ZoneId zone) { + public AthenzService getConfigServerAthenzIdentity(ZoneId zone) { return new AthenzService("vespadomain", "provider-" + zone.environment().value() + "-" + zone.region().value()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 513e5520d85..f642f3210b7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index de9da83826d..637203f87cc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.RotationStatus; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index 2b8e4f52d23..256ace4ae09 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -9,9 +9,9 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -117,7 +117,7 @@ public class DnsMaintainerTest { for (int i = 1; i <= staleTotal; i++) { Rotation r = rotation(i); tester.controllerTester().nameService().createCname(RecordName.from("stale-record-" + i + "." + - GlobalDnsName.OATH_DNS_SUFFIX), + Endpoint.OATH_DNS_SUFFIX), RecordData.from(r.name() + ".")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 86370980729..7a54936191a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 76fce662e27..7a008d1f478 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -6,8 +6,8 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; @@ -168,7 +168,7 @@ public class OsUpgraderTest { node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), node.canonicalFlavor(), node.clusterId(), node.clusterType())); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index abaaf3a5d86..d8e6f573592 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java index c3e8e63e805..0541a0b05f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java @@ -6,11 +6,12 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -56,15 +57,22 @@ public class RoutingPolicyMaintainerTest { public void maintains_global_routing_policies() { int clustersPerZone = 2; tester.deployCompletely(app1, applicationPackage); - Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0"))); + // Cluster is member of 2 global rotations + Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0"), RotationName.from("r1"))); provisionLoadBalancers(app1, clustersPerZone, rotations); - // Creates alias record for cluster0 + // Creates alias records for cluster0 maintainer.maintain(); - Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app1--tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app1.tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r1.app1.tenant1.global.vespa.oath.cloud")); assertEquals(2, records1.get().size()); + assertEquals(records1.get().size(), records2.get().size()); assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records1.get().get(0).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records2.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records2.get().get(1).data().asString()); + assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() + .endpointsIn(SystemName.main).asList().size()); // Applications gains a new deployment ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() @@ -85,13 +93,13 @@ public class RoutingPolicyMaintainerTest { assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(2).data().asString()); // Another application is deployed - Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app2--tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records3 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app2.tenant1.global.vespa.oath.cloud")); tester.deployCompletely(app2, applicationPackage); provisionLoadBalancers(app2, 1, Map.of(0, Set.of(RotationName.from("r0")))); maintainer.maintain(); - assertEquals(2, records2.get().size()); - assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records2.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records2.get().get(1).data().asString()); + assertEquals(2, records3.get().size()); + assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records3.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records3.get().get(1).data().asString()); // All rotations for app1 are removed provisionLoadBalancers(app1, clustersPerZone, Collections.emptyMap()); @@ -101,7 +109,7 @@ public class RoutingPolicyMaintainerTest { assertEquals(clustersPerZone * numberOfDeployments, policies.size()); assertTrue("Rotation membership is removed from all policies", policies.stream().allMatch(policy -> policy.rotations().isEmpty())); - assertEquals("Rotations for " + app2 + " are not removed", 2, records2.get().size()); + assertEquals("Rotations for " + app2 + " are not removed", 2, records3.get().size()); } @Test @@ -114,10 +122,10 @@ public class RoutingPolicyMaintainerTest { // Creates records and policies for all clusters in all zones maintainer.maintain(); Set<String> expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(4, policies(app1).size()); @@ -131,12 +139,12 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app1, clustersPerZone + 1); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(6, policies(app1).size()); @@ -146,16 +154,16 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app2, clustersPerZone); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(4, policies(app2).size()); @@ -164,14 +172,14 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app1, clustersPerZone); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); @@ -183,10 +191,10 @@ public class RoutingPolicyMaintainerTest { }); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index d832a75eaaa..ccc175402cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 2a13d73e42d..484516d7cd7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -8,7 +8,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index e0debf6d5db..d7c0ac9fe9e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index e329aee7fc6..4fe465ce01e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -5,7 +5,7 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 5243e8c2759..d9cdea2ea7b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -19,7 +19,7 @@ import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index f65cf6af346..11ac250d4e0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -10,7 +10,7 @@ import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java new file mode 100644 index 00000000000..95477758deb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -0,0 +1,88 @@ +package com.yahoo.vespa.hosted.controller.restapi; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.Roles; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; + +import java.nio.charset.StandardCharsets; +import java.security.Principal; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Controller container test with services.xml which accommodates cloud user management. + * + * @author jonmv + */ +public class ControllerContainerCloudTest extends ControllerContainerTest { + + @Override + protected SystemName system() { + return SystemName.Public; + } + + @Override + protected String variablePartXml() { + return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMarketplace'/>\n" + + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + + " <binding>http://*/application/v4/*</binding>\n" + + " <binding>http://*/api/application/v4/*</binding>\n" + + " </handler>\n" + + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.user.UserApiHandler'>\n" + + " <binding>http://*/user/v1/*</binding>\n" + + " <binding>http://*/api/user/v1/*</binding>\n" + + " </handler>\n" + + + " <http>\n" + + " <server id='default' port='8080' />\n" + + " <filtering>\n" + + " <request-chain id='default'>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + + " <binding>http://*/*</binding>\n" + + " </request-chain>\n" + + " </filtering>\n" + + " </http>\n"; + } + + protected static final String accessDenied = "{\n" + + " \"code\" : 403,\n" + + " \"message\" : \"Access denied\"\n" + + "}"; + + protected RequestBuilder request(String path) { return new RequestBuilder(path, Request.Method.GET); } + protected RequestBuilder request(String path, Request.Method method) { return new RequestBuilder(path, method); } + + protected class RequestBuilder implements Supplier<Request> { + private final String path; + private final Request.Method method; + private byte[] data = new byte[0]; + private Principal user = () -> "user@test"; + private Set<Role> roles = Set.of(new Roles(system()).everyone()); + + private RequestBuilder(String path, Request.Method method) { + this.path = path; + this.method = method; + } + + public RequestBuilder data(byte[] data) { this.data = data; return this; } + public RequestBuilder data(String data) { this.data = data.getBytes(StandardCharsets.UTF_8); return this; } + public RequestBuilder user(String user) { this.user = () -> user; return this; } + public RequestBuilder roles(Set<Role> roles) { this.roles = roles; return this; } + + @Override + public Request get() { + Request request = new Request("http://localhost:8080" + path, data, method, user); + request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(user, roles)); + request.getHeaders().put("Content-Type", "application/json"); + return request; + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index c21d4b4b0bf..6abfa7fa72d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -5,6 +5,7 @@ import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; @@ -36,112 +37,120 @@ public class ControllerContainerTest { protected JDisc container; @Before - public void startContainer() { container = JDisc.fromServicesXml(controllerServicesXml, Networking.disable); } + public void startContainer() { container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable); } @After public void stopContainer() { container.close(); } - private final String controllerServicesXml = - "<jdisc version='1.0'>\n" + - " <config name=\"container.handler.threadpool\">\n" + - " <maxthreads>10</maxthreads>\n" + - " </config> \n" + - " <config name='vespa.hosted.zone.config.zone'>\n" + - " <system>main</system>\n" + - " </config>\n" + - " <config name=\"vespa.hosted.rotation.config.rotations\">\n" + - " <rotations>\n" + - " <item key=\"rotation-id-1\">rotation-fqdn-1</item>\n" + - " <item key=\"rotation-id-2\">rotation-fqdn-2</item>\n" + - " <item key=\"rotation-id-3\">rotation-fqdn-3</item>\n" + - " <item key=\"rotation-id-4\">rotation-fqdn-4</item>\n" + - " <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" + - " </rotations>\n" + - " </config>\n" + - " <config name=\"jdisc.http.filter.security.cors.cors-filter\">\n" + - " <allowedUrls>\n" + - " <item>http://localhost</item>\n" + - " </allowedUrls>\n" + - " </config>\n" + - " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.restapi.cost.NoopCostReportConsumer'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'>\n" + - " <config name=\"vespa.hosted.controller.authority.config.api-authority\">\n" + - " <authorities><item>https://localhost:4443/</item></authorities>\n" + - " </config>" + - " </component>" + - " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.ApplicationStoreMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.role.Roles'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + - " <binding>http://*/application/v4/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + - " <binding>http://*/athenz/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.user.UserApiHandler'>\n" + - " <binding>http://*/user/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + - " <binding>http://*/deployment/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.BadgeApiHandler'>\n" + - " <binding>http://*/badge/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>\n" + - " <binding>http://*/controller/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + - " <binding>http://*/os/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" + - " <binding>http://*/cost/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + - " <binding>http://*/zone/v1</binding>\n" + - " <binding>http://*/zone/v1/*</binding>\n" + - " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" + - " <binding>http://*/zone/v2</binding>\n" + - " <binding>http://*/zone/v2/*</binding>\n" + - " </handler>\n" + - " <http>\n" + - " <server id='default' port='8080' />\n" + - " <filtering>\n" + - " <request-chain id='default'>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock'/>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.AthenzRoleFilter'/>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n" + - "</jdisc>"; + private String controllerServicesXml() { + return "<jdisc version='1.0'>\n" + + " <config name=\"container.handler.threadpool\">\n" + + " <maxthreads>10</maxthreads>\n" + + " </config> \n" + + " <config name=\"cloud.config.configserver\">\n" + + " <system>" + system().name() + "</system>\n" + + " </config> \n" + + " <config name=\"vespa.hosted.rotation.config.rotations\">\n" + + " <rotations>\n" + + " <item key=\"rotation-id-1\">rotation-fqdn-1</item>\n" + + " <item key=\"rotation-id-2\">rotation-fqdn-2</item>\n" + + " <item key=\"rotation-id-3\">rotation-fqdn-3</item>\n" + + " <item key=\"rotation-id-4\">rotation-fqdn-4</item>\n" + + " <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" + + " </rotations>\n" + + " </config>\n" + + " <config name=\"jdisc.http.filter.security.cors.cors-filter\">\n" + + " <allowedUrls>\n" + + " <item>http://localhost</item>\n" + + " </allowedUrls>\n" + + " </config>\n" + + " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.restapi.cost.NoopCostReportConsumer'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'>\n" + + " <config name=\"vespa.hosted.controller.authority.config.api-authority\">\n" + + " <authorities><item>https://localhost:4443/</item></authorities>\n" + + " </config>" + + " </component>" + + " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.integration.ApplicationStoreMock'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.role.Roles'/>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + + " <binding>http://*/deployment/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.BadgeApiHandler'>\n" + + " <binding>http://*/badge/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>\n" + + " <binding>http://*/controller/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + + " <binding>http://*/os/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" + + " <binding>http://*/cost/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>\n" + + " <binding>http://*/zone/v1</binding>\n" + + " <binding>http://*/zone/v1/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" + + " <binding>http://*/zone/v2</binding>\n" + + " <binding>http://*/zone/v2/*</binding>\n" + + " </handler>\n" + + variablePartXml() + + "</jdisc>"; + } + + protected SystemName system() { + return SystemName.main; + } + + protected String variablePartXml() { + return " <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" + + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + + " <binding>http://*/application/v4/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + + " <binding>http://*/athenz/v1/*</binding>\n" + + " </handler>\n" + + + " <http>\n" + + " <server id='default' port='8080' />\n" + + " <filtering>\n" + + " <request-chain id='default'>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock'/>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.AthenzRoleFilter'/>\n" + + " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + + " <binding>http://*/*</binding>\n" + + " </request-chain>\n" + + " </filtering>\n" + + " </http>\n"; + } protected void assertResponse(Request request, int responseStatus, String responseMessage) { Response response = container.handleRequest(request); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index bde1c037bf2..f617dc0a447 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -34,7 +34,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -380,6 +380,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .recursive("true"), new File("application1-recursive.json")); + // GET nodes + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/nodes", GET) + .userIdentity(USER_ID), + new File("application-nodes.json")); + // GET logs tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/prod/region/us-central-1/instance/default/logs?from=1233&to=3214", GET) .userIdentity(USER_ID), @@ -1327,7 +1332,8 @@ public class ApplicationApiTest extends ControllerContainerTest { public void applicationWithPerClusterGlobalRotation() { Application app = controllerTester.createApplication(); RoutingPolicy policy = new RoutingPolicy(app.id(), ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ClusterSpec.Id.from("default"), HostName.from("lb-0-canonical-name"), + ClusterSpec.Id.from("default"), controllerTester.controller().system(), + HostName.from("lb-0-canonical-name"), Optional.of("dns-zone-1"), Set.of(RotationName.from("c0"))); tester.controller().curator().writeRoutingPolicies(app.id(), Set.of(policy)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index df1f202a75c..212ee272a63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import org.json.JSONException; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java index b583aaedde1..9ef94cc9afd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json index cd531bb96da..baaf0cd038d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json @@ -8,7 +8,7 @@ "changeBlockers": [], "compileVersion": "(ignore)", "globalRotations": [ - "https://c0--application1--tenant1.global.vespa.oath.cloud:4443/" + "https://c0.application1.tenant1.global.vespa.oath.cloud/" ], "instances": [], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json new file mode 100644 index 00000000000..eb53ff7161e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -0,0 +1,13 @@ +{ + "nodes": [ + { + "hostname": "host-tenant1:application1:default-prod.us-central-1", + "state": "active", + "orchestration": "unorchestrated", + "version": "6.1", + "flavor": "d-2-8-50", + "clusterId": "cluster", + "clusterType": "container" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index f2f38f7f509..b52fec761d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -231,8 +231,6 @@ "changeBlockers": [], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 22e8573b1d4..4b2cb397b5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -219,8 +219,6 @@ ], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 662e045d169..fa903b61825 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -219,8 +219,6 @@ ], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java index 333d3155262..8cb439fee37 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index ddd8977c5bf..575427c9222 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index dc4235e52bf..b48cb4bff50 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.role.Roles; import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; -import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import org.junit.Before; @@ -51,7 +50,7 @@ public class AthenzRoleFilterTest { public void setup() { tester = new ControllerTester(); filter = new AthenzRoleFilter(new CorsFilterConfig.Builder().build(), - new AthenzFacade(new AthenzClientFactoryMock(tester.athenzDb())), + new AthenzClientFactoryMock(tester.athenzDb()), tester.controller()); tester.athenzDb().hostedOperators.add(HOSTED_OPERATOR.getIdentity()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 4ea4975f38c..bc8dd8d4479 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -9,8 +9,8 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; @@ -149,7 +149,7 @@ public class OsApiTest extends ControllerContainerTest { node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), node.canonicalFlavor(), node.clusterId(), node.clusterType())); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index d7fd38b5f41..3a78e9fc262 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -1,22 +1,162 @@ package com.yahoo.vespa.hosted.controller.restapi.user; -import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.Roles; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; import org.junit.Test; +import java.io.File; +import java.util.Set; + +import static com.yahoo.application.container.handler.Request.Method.DELETE; +import static com.yahoo.application.container.handler.Request.Method.POST; +import static com.yahoo.application.container.handler.Request.Method.PUT; +import static org.junit.Assert.assertEquals; + /** * @author jonmv */ -public class UserApiTest extends ControllerContainerTest { +public class UserApiTest extends ControllerContainerCloudTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/"; @Test - public void testUserApi() { - ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); + public void testUserManagement() { + ContainerTester tester = new ContainerTester(container, responseFiles); + assertEquals(SystemName.Public, tester.controller().system()); + Roles roles = new Roles(tester.controller().system()); + Set<Role> operator = Set.of(roles.hostedOperator()); + ApplicationId id = ApplicationId.from("my-tenant", "my-app", "default"); + + + // GET at application/v4 root fails as it's not public read. + tester.assertResponse(request("/application/v4/"), + accessDenied, 403); + + // GET at application/v4/tenant succeeds for operators. + tester.assertResponse(request("/application/v4/tenant") + .roles(operator), + "[]"); + + // GET at application/v4/tenant is available also under the /api prefix. + tester.assertResponse(request("/api/application/v4/tenant") + .roles(operator), + "[]"); + + // POST a tenant is not available to everyone. + tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) + .data("{\"token\":\"hello\"}"), + accessDenied, 403); + + // POST a tenant is available to operators. + tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) + .roles(operator) + .user("owner@tenant") + .data("{\"token\":\"hello\"}"), + new File("tenant-without-applications.json")); + + // PUT a tenant is not available to anyone. + tester.assertResponse(request("/application/v4/user/", PUT) + .roles(operator), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Not authenticated or not a user.\"}", 403); + + // GET at user/v1 root fails as no access control is defined there. + tester.assertResponse(request("/user/v1/"), + accessDenied, 403); + + // POST a hosted operator role is not allowed. + tester.assertResponse(request("/user/v1/tenant/my-tenant", POST) + .roles(Set.of(roles.tenantOwner(id.tenant()))) + .data("{\"user\":\"evil@evil\",\"roleName\":\"hostedOperator\"}"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Malformed or illegal role name 'hostedOperator'.\"}", 400); + + // POST a tenant operator is available to the tenant owner. + tester.assertResponse(request("/user/v1/tenant/my-tenant", POST) + .roles(Set.of(roles.tenantOwner(id.tenant()))) + .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"), + "{\"message\":\"user 'operator@tenant' is now a member of role 'tenantOperator' of 'my-tenant'\"}"); + + // POST a tenant admin is not available to a tenant operator. + tester.assertResponse(request("/user/v1/tenant/my-tenant", POST) + .roles(Set.of(roles.tenantOperator(id.tenant()))) + .data("{\"user\":\"admin@tenant\",\"roleName\":\"tenantAdmin\"}"), + accessDenied, 403); + + // POST an application admin for a non-existent application fails. + tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", POST) + .roles(Set.of(roles.tenantOwner(TenantName.from("my-tenant")))) + .data("{\"user\":\"admin@app\",\"roleName\":\"applicationAdmin\"}"), + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"NullPointerException\"}", 500); + + // POST an application is allowed for a tenant operator. + tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", POST) + .user("operator@tenant") + .roles(Set.of(roles.tenantOperator(id.tenant()))), + new File("application-created.json")); + + // POST an application is not allowed under a different tenant. + tester.assertResponse(request("/application/v4/tenant/other-tenant/application/my-app", POST) + .roles(Set.of(roles.tenantOperator(id.tenant()))), + accessDenied, 403); + + // POST an application role is allowed for a tenant admin. + tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", POST) + .roles(Set.of(roles.tenantAdmin(id.tenant()))) + .data("{\"user\":\"reader@app\",\"roleName\":\"applicationReader\"}"), + "{\"message\":\"user 'reader@app' is now a member of role 'applicationReader' of 'my-app' owned by 'my-tenant'\"}"); + + // POST a tenant role is not allowed to an application. + tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", POST) + .roles(Set.of(roles.hostedOperator())) + .data("{\"user\":\"reader@app\",\"roleName\":\"tenantOperator\"}"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Malformed or illegal role name 'tenantOperator'.\"}", 400); + + // GET tenant role information is available to application readers. + tester.assertResponse(request("/user/v1/tenant/my-tenant") + .roles(Set.of(roles.applicationReader(id.tenant(), id.application()))), + new File("tenant-roles.json")); + + // GET application role information is available to tenant operators. + tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app") + .roles(Set.of(roles.tenantOperator(id.tenant()))), + new File("application-roles.json")); + + // GET application role information is available also under the /api prefix. + tester.assertResponse(request("/api/user/v1/tenant/my-tenant/application/my-app") + .roles(Set.of(roles.tenantOperator(id.tenant()))), + new File("application-roles.json")); + + // DELETE an application role is allowed for an application admin. + tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE) + .roles(Set.of(roles.applicationAdmin(id.tenant(), id.application()))) + .data("{\"user\":\"operator@tenant\",\"roleName\":\"applicationAdmin\"}"), + "{\"message\":\"user 'operator@tenant' is no longer a member of role 'applicationAdmin' of 'my-app' owned by 'my-tenant'\"}"); + + // DELETE an application is available to application admins. + tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", DELETE) + .roles(Set.of(roles.applicationAdmin(id.tenant(), id.application()))), + ""); + + // DELETE a tenant role is available to tenant admins. + tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) + .roles(Set.of(roles.tenantAdmin(id.tenant()))) + .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"), + "{\"message\":\"user 'operator@tenant' is no longer a member of role 'tenantOperator' of 'my-tenant'\"}"); + + // DELETE the last tenant owner is not allowed. + tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) + .roles(operator) + .data("{\"user\":\"owner@tenant\",\"roleName\":\"tenantOwner\"}"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can't remove the last owner of a tenant.\"}", 400); - tester.assertResponse(authenticatedRequest("http://localhost:8080/user/v1/"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"No 'GET' handler at '/user/v1/'\"}", 404); + // DELETE the tenant is available to the tenant owner. + tester.assertResponse(request("/application/v4/tenant/my-tenant", DELETE) + .roles(Set.of(roles.tenantOwner(id.tenant()))), + new File("tenant-without-applications.json")); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json new file mode 100644 index 00000000000..2a779f0ee55 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json @@ -0,0 +1,6 @@ +{ + "tenant": "my-tenant", + "application": "my-app", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json new file mode 100644 index 00000000000..e6ecf86c4ec --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-roles.json @@ -0,0 +1,54 @@ +{ + "tenant": "my-tenant", + "application": "my-app", + "roleNames": [ + "applicationAdmin", + "applicationOperator", + "applicationDeveloper", + "applicationReader" + ], + "users": [ + { + "name": "reader@app", + "roles": { + "applicationAdmin": { + "explicit": false, + "implied": false + }, + "applicationOperator": { + "explicit": false, + "implied": false + }, + "applicationDeveloper": { + "explicit": false, + "implied": false + }, + "applicationReader": { + "explicit": true, + "implied": false + } + } + }, + { + "name": "operator@tenant", + "roles": { + "applicationAdmin": { + "explicit": true, + "implied": false + }, + "applicationOperator": { + "explicit": false, + "implied": true + }, + "applicationDeveloper": { + "explicit": false, + "implied": true + }, + "applicationReader": { + "explicit": false, + "implied": true + } + } + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json new file mode 100644 index 00000000000..46f715b481b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-roles.json @@ -0,0 +1,44 @@ +{ + "tenant": "my-tenant", + "roleNames": [ + "tenantOwner", + "tenantAdmin", + "tenantOperator" + ], + "users": [ + { + "name": "operator@tenant", + "roles": { + "tenantOwner": { + "explicit": false, + "implied": false + }, + "tenantAdmin": { + "explicit": false, + "implied": false + }, + "tenantOperator": { + "explicit": true, + "implied": false + } + } + }, + { + "name": "owner@tenant", + "roles": { + "tenantOwner": { + "explicit": true, + "implied": false + }, + "tenantAdmin": { + "explicit": false, + "implied": true + }, + "tenantOperator": { + "explicit": false, + "implied": true + } + } + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json new file mode 100644 index 00000000000..e4ca5a02446 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -0,0 +1,5 @@ +{ + "tenant": "my-tenant", + "type": "CLOUD", + "applications": [] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index a7832ccbc1f..9c853f211ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index 8f056f1710f..eff212b6c16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.text.Utf8; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 666c7774cf5..8d1f40260e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -65,8 +65,8 @@ public class RotationRepositoryTest { application = tester.applications().require(application.id()); assertEquals(expected.id(), application.rotation().get()); - assertEquals(URI.create("http://app1.tenant1.global.vespa.yahooapis.com:4080/"), - application.globalDnsName(SystemName.main).get().url()); + assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), + application.endpointsIn(SystemName.main).main().get().url()); try (RotationLock lock = repository.lock()) { Rotation rotation = repository.getOrAssignRotation(tester.applications().require(application.id()), lock); assertEquals(expected, rotation); @@ -153,10 +153,9 @@ public class RotationRepositoryTest { Application application = tester.createApplication("app2", "tenant2", 22L, 2L); tester.deployCompletely(application, applicationPackage); - assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()) - .rotation().get()); - assertEquals("https://cd--app2--tenant2.global.vespa.yahooapis.com:4443/", tester.applications().require(application.id()) - .globalDnsName(SystemName.cd).get().secureUrl().toString()); + assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()).rotation().get()); + assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", tester.applications().require(application.id()) + .endpointsIn(SystemName.cd).main().get().url().toString()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index c74081bee63..a365285b752 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index 07efd419c97..ec74fb9246d 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -31,7 +31,13 @@ import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.statistics.Statistics; import java.util.TimerTask; -import java.util.concurrent.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; @@ -203,8 +209,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return null; } - @SuppressWarnings("unchecked") - void submit(DocumentProcessingTask task) { + private void submit(DocumentProcessingTask task) { if (threadPool.isAboveLimit()) { task.queueFull(); } else { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 0bc95fe6c7b..ca4648678a5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -17,8 +17,6 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -34,7 +32,6 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask private final DocumentProcessingHandler docprocHandler; private RequestContext requestContext; - private int waitCounter; private final static AtomicLong seq = new AtomicLong(); private final long seqNum; @@ -45,7 +42,6 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask seqNum = seq.getAndIncrement(); this.requestContext = requestContext; this.docprocHandler = docprocHandler; - this.waitCounter = 10; this.service = service; } @@ -80,18 +76,6 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask } /** - * Used by DocprocThreadManager. If a ProcessingTask has been taken by a thread, it can wait() no longer than - * waitCounter (currently 10) times before being executed. This is to prevent large tasks from being delayed - * forever. - * - * @return true if this task can wait, false if it must run NOW. - */ - boolean doWait() { - --waitCounter; - return (waitCounter > 0); - } - - /** * Processes a single Processing, and fails the message if this processing fails. * * @param executor the DocprocService to use for processing diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index edb426c6392..ff891dbb298 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -1,18 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessException; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.NetworkMessageBus; import com.yahoo.messagebus.RPCMessageBus; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.local.LocalNetwork; -import com.yahoo.messagebus.network.local.LocalWire; -import com.yahoo.messagebus.routing.RoutingTable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java index 6bd5d658509..0c40f527287 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi.messagebus.loadtypes; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import java.security.MessageDigest; diff --git a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp index c3b42124155..aa4c3b8c021 100644 --- a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp +++ b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp @@ -4,6 +4,8 @@ #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/operation.h> #include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/data/memory.h> +#include <vespa/vespalib/objects/nbostream.h> #include <iostream> using namespace vespalib::eval; @@ -12,6 +14,8 @@ using Cell = SimpleTensor::Cell; using Cells = SimpleTensor::Cells; using Address = SimpleTensor::Address; using Stash = vespalib::Stash; +using vespalib::nbostream; +using vespalib::Memory; TensorSpec to_spec(const Value &a) { return SimpleTensorEngine::ref().to_spec(a); } @@ -143,4 +147,208 @@ TEST("require that simple tensors support dimension reduction") { EXPECT_NOT_EQUAL(to_spec(*result_sum_y), to_spec(*result_sum_x)); } +//----------------------------------------------------------------------------- + +struct SparseTensorExample { + TensorSpec make_spec() const { + return TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","a"}}, 1) + .add({{"x","a"},{"y","b"}}, 2) + .add({{"x","b"},{"y","a"}}, 3); + } + std::unique_ptr<SimpleTensor> make_tensor() const { + return SimpleTensor::create(make_spec()); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 2; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 3; + } + void encode_default(nbostream &dst) const { + dst.putInt1_4Bytes(1); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST_F("require that sparse tensors can be decoded", SparseTensorExample()) { + nbostream data1; + nbostream data2; + nbostream data3; + f1.encode_default(data1); + f1.encode_with_double(data2); + f1.encode_with_float(data3); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data1)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data2)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data3)), f1.make_spec()); +} + +TEST_F("require that sparse tensors can be encoded", SparseTensorExample()) { + nbostream data; + nbostream expect; + SimpleTensor::encode(*f1.make_tensor(), data); + f1.encode_default(expect); + EXPECT_EQUAL(Memory(data.peek(), data.size()), Memory(expect.peek(), expect.size())); +} + +//----------------------------------------------------------------------------- + +struct DenseTensorExample { + TensorSpec make_spec() const { + return TensorSpec("tensor(x[3],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 2) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 4) + .add({{"x",2},{"y",0}}, 5) + .add({{"x",2},{"y",1}}, 6); + } + std::unique_ptr<SimpleTensor> make_tensor() const { + return SimpleTensor::create(make_spec()); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(2); + dst << (T) 1; + dst << (T) 2; + dst << (T) 3; + dst << (T) 4; + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const { + dst.putInt1_4Bytes(2); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST_F("require that dense tensors can be decoded", DenseTensorExample()) { + nbostream data1; + nbostream data2; + nbostream data3; + f1.encode_default(data1); + f1.encode_with_double(data2); + f1.encode_with_float(data3); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data1)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data2)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data3)), f1.make_spec()); +} + +TEST_F("require that dense tensors can be encoded", DenseTensorExample()) { + nbostream data; + nbostream expect; + SimpleTensor::encode(*f1.make_tensor(), data); + f1.encode_default(expect); + EXPECT_EQUAL(Memory(data.peek(), data.size()), Memory(expect.peek(), expect.size())); +} + +//----------------------------------------------------------------------------- + +struct MixedTensorExample { + TensorSpec make_spec() const { + return TensorSpec("tensor(x{},y{},z[2])") + .add({{"x","a"},{"y","a"},{"z",0}}, 1) + .add({{"x","a"},{"y","a"},{"z",1}}, 2) + .add({{"x","a"},{"y","b"},{"z",0}}, 3) + .add({{"x","a"},{"y","b"},{"z",1}}, 4) + .add({{"x","b"},{"y","a"},{"z",0}}, 5) + .add({{"x","b"},{"y","a"},{"z",1}}, 6); + } + std::unique_ptr<SimpleTensor> make_tensor() const { + return SimpleTensor::create(make_spec()); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(1); + dst.writeSmallString("z"); + dst.putInt1_4Bytes(2); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst << (T) 2; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 3; + dst << (T) 4; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const { + dst.putInt1_4Bytes(3); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST_F("require that mixed tensors can be decoded", MixedTensorExample()) { + nbostream data1; + nbostream data2; + nbostream data3; + f1.encode_default(data1); + f1.encode_with_double(data2); + f1.encode_with_float(data3); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data1)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data2)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data3)), f1.make_spec()); +} + +TEST_F("require that mixed tensors can be encoded", MixedTensorExample()) { + nbostream data; + nbostream expect; + SimpleTensor::encode(*f1.make_tensor(), data); + f1.encode_default(expect); + EXPECT_EQUAL(Memory(data.peek(), data.size()), Memory(expect.peek(), expect.size())); +} + +//----------------------------------------------------------------------------- + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index ffdc601932e..f7db7816fad 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -268,6 +268,8 @@ TEST("require that value type spec can be parsed") { EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor(y[10])")); EXPECT_EQUAL(ValueType::tensor_type({{"z", 0}}), ValueType::from_spec("tensor(z[])")); EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 0}}), ValueType::from_spec("tensor(x{},y[10],z[])")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor<double>(y[10])")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor<float>(y[10])")); } TEST("require that value type spec can be parsed with extra whitespace") { @@ -280,6 +282,8 @@ TEST("require that value type spec can be parsed with extra whitespace") { EXPECT_EQUAL(ValueType::tensor_type({{"z", 0}}), ValueType::from_spec(" tensor ( z [ ] ) ")); EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 0}}), ValueType::from_spec(" tensor ( x { } , y [ 10 ] , z [ ] ) ")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec(" tensor < double > ( y [ 10 ] ) ")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec(" tensor < float > ( y [ 10 ] ) ")); } TEST("require that malformed value type spec is parsed as error") { @@ -300,6 +304,7 @@ TEST("require that malformed value type spec is parsed as error") { EXPECT_TRUE(ValueType::from_spec("tensor(x{},x{})").is_error()); EXPECT_TRUE(ValueType::from_spec("tensor(x{},x[10])").is_error()); EXPECT_TRUE(ValueType::from_spec("tensor(x{},x[])").is_error()); + EXPECT_TRUE(ValueType::from_spec("tensor<float16>(x[10])").is_error()); } struct ParseResult { diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 1836f2088f3..1bce8983666 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -19,6 +19,9 @@ using CellRef = std::reference_wrapper<const Cell>; namespace { +constexpr uint32_t DOUBLE_CELL_TYPE = 0; +constexpr uint32_t FLOAT_CELL_TYPE = 1; + void assert_type(const ValueType &type) { (void) type; assert(!type.is_abstract()); @@ -418,14 +421,17 @@ public: struct Format { bool is_sparse; bool is_dense; + bool with_cell_type; uint32_t tag; explicit Format(const TypeMeta &meta) : is_sparse(meta.mapped.size() > 0), is_dense((meta.indexed.size() > 0) || !is_sparse), + with_cell_type(false), tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0)) {} explicit Format(uint32_t tag_in) : is_sparse((tag_in & 0x1) != 0), is_dense((tag_in & 0x2) != 0), + with_cell_type((tag_in & 0x4) != 0), tag(tag_in) {} ~Format() {} }; @@ -458,6 +464,13 @@ void encode_mapped_labels(nbostream &output, const TypeMeta &meta, const Address } } +uint32_t maybe_decode_cell_type(nbostream &input, const Format &format) { + if (format.with_cell_type) { + return input.getInt1_4Bytes(); + } + return DOUBLE_CELL_TYPE; +} + ValueType decode_type(nbostream &input, const Format &format) { std::vector<ValueType::Dimension> dim_list; if (format.is_sparse) { @@ -496,17 +509,20 @@ void decode_mapped_labels(nbostream &input, const TypeMeta &meta, Address &addr) } } -void decode_cells(nbostream &input, const ValueType &type, const TypeMeta meta, +void decode_cells(uint32_t cell_type, nbostream &input, const ValueType &type, const TypeMeta meta, Address &address, size_t n, Builder &builder) { if (n < meta.indexed.size()) { Label &label = address[meta.indexed[n]]; size_t size = type.dimensions()[meta.indexed[n]].size; for (label.index = 0; label.index < size; ++label.index) { - decode_cells(input, type, meta, address, n + 1, builder); + decode_cells(cell_type, input, type, meta, address, n + 1, builder); } } else { - builder.set(address, input.readValue<double>()); + double value = (cell_type == FLOAT_CELL_TYPE) + ? input.readValue<float>() + : input.readValue<double>(); + builder.set(address, value); } } @@ -693,6 +709,7 @@ std::unique_ptr<SimpleTensor> SimpleTensor::decode(nbostream &input) { Format format(input.getInt1_4Bytes()); + uint32_t cell_type = maybe_decode_cell_type(input, format); ValueType type = decode_type(input, format); TypeMeta meta(type); Builder builder(type); @@ -700,7 +717,7 @@ SimpleTensor::decode(nbostream &input) Address address(type.dimensions().size(), Label(size_t(0))); for (size_t i = 0; i < num_blocks; ++i) { decode_mapped_labels(input, meta, address); - decode_cells(input, type, meta, address, 0, builder); + decode_cells(cell_type, input, type, meta, address, 0, builder); } return builder.build(); } diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index 229a9201f08..6076988ad50 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -12,6 +12,14 @@ namespace { class ParseContext { +public: + struct Mark { + const char *pos; + char curr; + bool failed; + Mark(const char *pos_in, char curr_in, bool failed_in) + : pos(pos_in), curr(curr_in), failed(failed_in) {} + }; private: const char *_pos; const char *_end; @@ -34,6 +42,14 @@ public: _pos_after = nullptr; } } + Mark mark() const { + return Mark(_pos, _curr, _failed); + } + void revert(Mark mark) { + _pos = mark.pos; + _curr = mark.curr; + _failed = mark.failed; + } void fail() { _failed = true; _curr = 0; @@ -77,6 +93,7 @@ vespalib::string parse_ident(ParseContext &ctx) { } size_t parse_int(ParseContext &ctx) { + ctx.skip_spaces(); vespalib::string num; for (; isdigit(ctx.get()); ctx.next()) { num.push_back(ctx.get()); @@ -91,11 +108,11 @@ ValueType::Dimension parse_dimension(ParseContext &ctx) { ValueType::Dimension dimension(parse_ident(ctx)); ctx.skip_spaces(); if (ctx.get() == '{') { - ctx.next(); // '{' + ctx.eat('{'); ctx.skip_spaces(); ctx.eat('}'); } else if (ctx.get() == '[') { - ctx.next(); // '[' + ctx.eat('['); ctx.skip_spaces(); if (ctx.get() == ']') { dimension.size = 0; @@ -129,6 +146,20 @@ std::vector<ValueType::Dimension> parse_dimension_list(ParseContext &ctx) { return list; } +vespalib::string parse_cell_type(ParseContext &ctx) { + auto mark = ctx.mark(); + ctx.skip_spaces(); + ctx.eat('<'); + auto cell_type = parse_ident(ctx); + ctx.skip_spaces(); + ctx.eat('>'); + if (ctx.failed()) { + ctx.revert(mark); + cell_type = "double"; + } + return cell_type; +} + } // namespace vespalib::eval::value_type::<anonymous> ValueType @@ -143,6 +174,10 @@ parse_spec(const char *pos_in, const char *end_in, const char *&pos_out) } else if (type_name == "double") { return ValueType::double_type(); } else if (type_name == "tensor") { + vespalib::string cell_type = parse_cell_type(ctx); + if ((cell_type != "double") && (cell_type != "float")) { + ctx.fail(); + } std::vector<ValueType::Dimension> list = parse_dimension_list(ctx); if (!ctx.failed()) { return ValueType::tensor_type(std::move(list)); diff --git a/eval/src/vespa/eval/tensor/serialization/format.txt b/eval/src/vespa/eval/tensor/serialization/format.txt index 780f88af01a..1a454b0ccf8 100644 --- a/eval/src/vespa/eval/tensor/serialization/format.txt +++ b/eval/src/vespa/eval/tensor/serialization/format.txt @@ -4,15 +4,25 @@ interpreted as a single unified binary format. The description below uses data types defined by document serialization (nbostream) combined with some comments and python-inspired flow-control. The mixed[3] binary format is defined in such a way that it overlays as -effortlessly as possible with both existing formats. +effortlessly as possible with the sparse[1] and dense[2] formats. + +All format archetypes can also be encoded with values other than +double. Using a separate bit to specify that the format includes cells +with a specific type gives rise to 3 new formats: +sparse_with_cell_type[5], dense_with_cell_type[6] and +mixed_with_cell_type[7]. //----------------------------------------------------------------------------- -1_4_int: type (1:sparse, 2:dense, 3:mixed) +1_4_int: type (1/5:sparse, 2/6:dense, 3/7:mixed) bit 0 -> 'sparse' bit 1 -> 'dense' + bit 2 -> 'with_cell_type' (mixed tensors are tagged as both 'sparse' and 'dense') +if ('with_cell_type') + 1_4_int -> cell_type (0:double, 1:float) + if ('sparse'): 1_4_int: number of mapped dimensions -> 'n_mapped' 'n_mapped' times: (sorted by dimension name) @@ -33,10 +43,16 @@ else: 'n_mapped' times: small_string: dimension label (same order as dimension names) prod('size_i') times: (product of all indexed dimension sizes) - double: cell value (last indexed dimension is nested innermost) + cell_type: cell value (last indexed dimension is nested innermost) //----------------------------------------------------------------------------- +Note: The mixed_with_cell_type format can be used to encode any +tensor. + +Note: cell_type defaults to double, but can be overridden by using any +of the '_with_cell_type' formats. + Note: A tensor with no dimensions should not be serialized as sparse[1], but when it is, it will contain an integer indicating the number of cells. diff --git a/functions.cmake b/functions.cmake index bfb4c945c16..2b460c8c5ea 100644 --- a/functions.cmake +++ b/functions.cmake @@ -566,6 +566,22 @@ function(install_symlink TARGET LINK) install_absolute_symlink(${CMAKE_INSTALL_PREFIX}/${TARGET} ${LINK}) endfunction(install_symlink) +function(install_configserver_component NAME) + cmake_parse_arguments( + PARAM + "" + "CLASSIFIER" + "" + ${ARGN} + ) + if(NOT PARAM_CLASSIFIER) + SET(PARAM_CLASSIFIER "jar-with-dependencies") + endif() + install(FILES "target/${NAME}-${PARAM_CLASSIFIER}.jar" DESTINATION lib/jars/) + install(DIRECTORY DESTINATION conf/configserver-app/components) + install_symlink(lib/jars/${NAME}-${PARAM_CLASSIFIER}.jar conf/configserver-app/components/${NAME}.jar) +endfunction() + function(add_extra_projects) if(EXTRA_PROJECTS) foreach(PROJECT ${EXTRA_PROJECTS}) diff --git a/http-utils/OWNERS b/http-utils/OWNERS new file mode 100644 index 00000000000..569bf1cc3a1 --- /dev/null +++ b/http-utils/OWNERS @@ -0,0 +1 @@ +bjorncs diff --git a/http-utils/README.md b/http-utils/README.md new file mode 100644 index 00000000000..8bbe2cd7d05 --- /dev/null +++ b/http-utils/README.md @@ -0,0 +1 @@ +# Http utilities for Java
\ No newline at end of file diff --git a/http-utils/pom.xml b/http-utils/pom.xml new file mode 100644 index 00000000000..aea402aef87 --- /dev/null +++ b/http-utils/pom.xml @@ -0,0 +1,52 @@ +<?xml version="1.0"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>http-utils</artifactId> + <packaging>jar</packaging> + <version>7-SNAPSHOT</version> + <dependencies> + <!-- provided --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <!-- compile scope --> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <scope>compile</scope> + </dependency> + + <!-- test scope --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> diff --git a/security-utils/src/main/java/com/yahoo/security/tls/https/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java index 9fa51fc36cb..5e7a9441fc8 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/https/VespaHttpClientBuilder.java +++ b/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java @@ -1,5 +1,5 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.security.tls.https; +package ai.vespa.util.http; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TlsContext; @@ -8,10 +8,15 @@ import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import javax.net.ssl.SSLParameters; @@ -34,19 +39,33 @@ public class VespaHttpClientBuilder { private static final Logger log = Logger.getLogger(VespaHttpClientBuilder.class.getName()); public interface ConnectionManagerFactory { - HttpClientConnectionManager create(SSLConnectionSocketFactory sslSocketFactory); + HttpClientConnectionManager create(Registry<ConnectionSocketFactory> socketFactoryRegistry); } private VespaHttpClientBuilder() {} + /** + * Create a client builder with default connection manager. + */ public static HttpClientBuilder create() { return createBuilder(null); } + /** + * Create a client builder with a user specified connection manager. + */ public static HttpClientBuilder create(ConnectionManagerFactory connectionManagerFactory) { return createBuilder(connectionManagerFactory); } + /** + * Creates a client builder with a {@link BasicHttpClientConnectionManager} configured. + * This connection manager uses a single connection for all requests. See Javadoc for details. + */ + public static HttpClientBuilder createWithBasicConnectionManager() { + return createBuilder(BasicHttpClientConnectionManager::new); + } + private static HttpClientBuilder createBuilder(ConnectionManagerFactory connectionManagerFactory) { var builder = HttpClientBuilder.create(); addSslSocketFactory(builder, connectionManagerFactory); @@ -60,7 +79,7 @@ public class VespaHttpClientBuilder { log.log(Level.FINE, "Adding ssl socket factory to client"); SSLConnectionSocketFactory socketFactory = createSslSocketFactory(tlsContext); if (connectionManagerFactory != null) { - builder.setConnectionManager(connectionManagerFactory.create(socketFactory)); + builder.setConnectionManager(connectionManagerFactory.create(createRegistry(socketFactory))); } else { builder.setSSLSocketFactory(socketFactory); } @@ -80,6 +99,13 @@ public class VespaHttpClientBuilder { return new SSLConnectionSocketFactory(tlsContext.context(), parameters.getProtocols(), parameters.getCipherSuites(), new NoopHostnameVerifier()); } + private static Registry<ConnectionSocketFactory> createRegistry(SSLConnectionSocketFactory sslSocketFactory) { + return RegistryBuilder.<ConnectionSocketFactory>create() + .register("https", sslSocketFactory) + .register("http", PlainConnectionSocketFactory.getSocketFactory()) + .build(); + } + static class HttpToHttpsRewritingRequestInterceptor implements HttpRequestInterceptor { @Override public void process(HttpRequest request, HttpContext context) { diff --git a/security-utils/src/test/java/com/yahoo/security/tls/https/VespaHttpClientBuilderTest.java b/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java index 10b8458359c..7ffd0e459b0 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/https/VespaHttpClientBuilderTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.security.tls.https; +package ai.vespa.util.http; -import com.yahoo.security.tls.https.VespaHttpClientBuilder.HttpToHttpsRewritingRequestInterceptor; +import ai.vespa.util.http.VespaHttpClientBuilder.HttpToHttpsRewritingRequestInterceptor; import org.apache.http.client.methods.HttpGet; import org.apache.http.protocol.BasicHttpContext; import org.junit.Test; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java index b6591ef4825..1e0d90e71f9 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java @@ -33,6 +33,9 @@ public class BindingRepository<T> implements Iterable<Map.Entry<UriPattern, T>> * @throws IllegalArgumentException If the URI pattern string could not be parsed. */ public void bind(String uriPattern, T target) { + if (uriPattern.startsWith("https://")) { + log.warning(() -> String.format("For binding '%s': 'https' is deprecated, use 'http' to match both 'http' and 'https'", uriPattern)); + } put(new UriPattern(uriPattern), target); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 5c3298a7aff..9a10c70ceab 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -87,6 +87,14 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog if (clientCert != null && clientCert.length > 0) { accessLogEntry.setSslPrincipal(clientCert[0].getSubjectX500Principal()); } + String sslSessionId = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_SSL_SESSION_ID); + if (sslSessionId != null) { + accessLogEntry.addKeyValue("ssl-session-id", sslSessionId); + } + String cipherSuite = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_CIPHER_SUITE); + if (cipherSuite != null) { + accessLogEntry.addKeyValue("cipher-suite", cipherSuite); + } final long startTime = request.getTimeStamp(); final long endTime = System.currentTimeMillis(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java index 2eb7f432ec2..65c8e153164 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java @@ -39,6 +39,8 @@ public class ServletRequest extends HttpServletRequestWrapper implements Servlet public static final String JDISC_REQUEST_PRINCIPAL = "jdisc.request.principal"; public static final String JDISC_REQUEST_X509CERT = "jdisc.request.X509Certificate"; public static final String SERVLET_REQUEST_X509CERT = "javax.servlet.request.X509Certificate"; + public static final String SERVLET_REQUEST_SSL_SESSION_ID = "javax.servlet.request.ssl_session_id"; + public static final String SERVLET_REQUEST_CIPHER_SUITE = "javax.servlet.request.cipher_suite"; private final HttpServletRequest request; private final HeaderFields headerFields; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index d36457cf42d..704a73710cf 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -4,7 +4,11 @@ package com.yahoo.messagebus.shared; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.IntermediateSessionParams; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; diff --git a/jrt/pom.xml b/jrt/pom.xml index e9383654e30..5208c0417cc 100644 --- a/jrt/pom.xml +++ b/jrt/pom.xml @@ -34,16 +34,6 @@ <artifactId>security-utils</artifactId> <version>${project.version}</version> <scope>compile</scope> - <exclusions> - <exclusion> <!-- not needed --> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </exclusion> - <exclusion> <!-- not needed --> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <!-- required due to bug in maven dependency resolving - bouncycastle is compile scope in security-utils, yet it is not part of test scope here --> <groupId>org.bouncycastle</groupId> diff --git a/logd/src/logd/config_subscriber.cpp b/logd/src/logd/config_subscriber.cpp index 0a88db9772f..ce29c808742 100644 --- a/logd/src/logd/config_subscriber.cpp +++ b/logd/src/logd/config_subscriber.cpp @@ -133,13 +133,15 @@ std::unique_ptr<Forwarder> ConfigSubscriber::make_forwarder(Metrics& metrics) { std::unique_ptr<Forwarder> result; - if (_logserver_use_rpc) { - result = std::make_unique<RpcForwarder>(metrics, _forward_filter, _supervisor, _logserver_host, - _logserver_rpc_port, 60.0, 100); + if (_use_logserver) { + if (_logserver_use_rpc) { + result = std::make_unique<RpcForwarder>(metrics, _forward_filter, _supervisor, _logserver_host, + _logserver_rpc_port, 60.0, 100); + } else { + result = LegacyForwarder::to_logserver(metrics, _forward_filter, _logserver_host, _logserver_port); + } } else { - result = _use_logserver ? - LegacyForwarder::to_logserver(metrics, _forward_filter, _logserver_host, _logserver_port) : - LegacyForwarder::to_dev_null(metrics); + result = LegacyForwarder::to_dev_null(metrics); } _need_new_forwarder = false; return result; diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index 288bd6a1eda..e8e752127b1 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -576,7 +576,6 @@ ], "methods": [ "public void <init>()", - "public void addRecurrentTask(com.yahoo.messagebus.Messenger$Task)", "public void start()", "public void deliverMessage(com.yahoo.messagebus.Message, com.yahoo.messagebus.MessageHandler)", "public void deliverReply(com.yahoo.messagebus.Reply, com.yahoo.messagebus.ReplyHandler)", diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 26e61e8917b..0f0b704bba7 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -58,13 +58,13 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, private static Logger log = Logger.getLogger(MessageBus.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final ProtocolRepository protocolRepository = new ProtocolRepository(); - private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<Map<String, RoutingTable>>(null); - private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<String, MessageHandler>(); + private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<>(null); + private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<>(); private final Network net; private final Messenger msn; private final Resender resender; - private int maxPendingCount = 0; - private int maxPendingSize = 0; + private int maxPendingCount; + private int maxPendingSize; private int pendingCount = 0; private int pendingSize = 0; private final Thread careTaker = new Thread(this::sendBlockedMessages); @@ -440,7 +440,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, @Override public void setupRouting(RoutingSpec spec) { - Map<String, RoutingTable> tables = new HashMap<String, RoutingTable>(); + Map<String, RoutingTable> tables = new HashMap<>(); for (int i = 0, len = spec.getNumTables(); i < len; ++i) { RoutingTableSpec table = spec.getTable(i); String name = table.getProtocol(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 4fb83386231..63e5dbb2d04 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -44,7 +44,7 @@ public class Messenger implements Runnable { * * @param task The task to add. */ - public void addRecurrentTask(final Task task) { + void addRecurrentTask(final Task task) { children.add(task); } @@ -159,7 +159,7 @@ public class Messenger implements Runnable { synchronized (this) { if (queue.isEmpty()) { try { - wait(100); + wait(10); } catch (final InterruptedException e) { continue; } @@ -210,13 +210,13 @@ public class Messenger implements Runnable { /** * <p>This method is called when being executed.</p> */ - public void run(); + void run(); /** * <p>This method is called for all tasks, even if {@link #run()} was * never called.</p> */ - public void destroy(); + void destroy(); } private static class MessageTask implements Runnable { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 64b7c4cb12e..e04cccfcbd1 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -1,15 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.routing; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.SendProxy; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.TraceLevel; +import com.yahoo.messagebus.TraceNode; import com.yahoo.messagebus.metrics.RouteMetricSet; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.ServiceAddress; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Stack; import java.util.concurrent.atomic.AtomicInteger; /** @@ -32,7 +45,7 @@ public class RoutingNode implements ReplyHandler { private final AtomicInteger pending = new AtomicInteger(0); private final Message msg; private Reply reply = null; - private Route route = null; + private Route route; private RoutingPolicy policy = null; private RoutingContext routingContext = null; private ServiceAddress serviceAddress = null; @@ -122,7 +135,7 @@ public class RoutingNode implements ReplyHandler { * * @param msg The error message to assign. */ - public void notifyAbort(String msg) { + private void notifyAbort(String msg) { Stack<RoutingNode> stack = new Stack<>(); stack.push(this); while (!stack.isEmpty()) { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java index 0ac962ad947..2ebe4103713 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java @@ -4,8 +4,17 @@ package com.yahoo.messagebus.routing; import com.yahoo.component.Vtag; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; @@ -23,7 +32,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 10ac30d8715..1811fc0c8f0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import java.net.URI; @@ -16,12 +17,12 @@ import java.util.stream.Collectors; */ public class ConfigServerInfo { private final URI loadBalancerEndpoint; - private final AthenzService configServerIdentity; + private final AthenzIdentity configServerIdentity; private final Function<String, URI> configServerHostnameToUriMapper; private final List<URI> configServerURIs; public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, - String scheme, int port, AthenzService configServerAthenzIdentity) { + String scheme, int port, AthenzIdentity configServerAthenzIdentity) { this.loadBalancerEndpoint = createLoadBalancerEndpoint(loadBalancerHostName, scheme, port); this.configServerIdentity = configServerAthenzIdentity; this.configServerHostnameToUriMapper = hostname -> URI.create(scheme + "://" + hostname + ":" + port); @@ -46,7 +47,7 @@ public class ConfigServerInfo { return loadBalancerEndpoint; } - public AthenzService getConfigServerIdentity() { + public AthenzIdentity getConfigServerIdentity() { return configServerIdentity; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 4fe0f420f05..550d6e7021e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -8,6 +8,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; @@ -63,7 +64,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final URI ztsEndpoint; private final Path trustStorePath; - private final AthenzService configserverIdentity; + private final AthenzIdentity configserverIdentity; private final Clock clock; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index 496f4bd667d..205e7b1e258 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; @@ -33,7 +34,7 @@ public interface NodeAgentContext extends TaskContext { return node().getNodeType(); } - AthenzService identity(); + AthenzIdentity identity(); DockerNetworking dockerNetworking(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 9ca19a76706..1b33fed151e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -4,6 +4,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; @@ -30,7 +31,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final NodeSpec node; private final Acl acl; private final ContainerName containerName; - private final AthenzService identity; + private final AthenzIdentity identity; private final DockerNetworking dockerNetworking; private final ZoneId zoneId; private final Path pathToNodeRootOnHost; @@ -38,7 +39,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final String vespaUser; private final String vespaUserOnHost; - public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzService identity, + public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, DockerNetworking dockerNetworking, ZoneId zoneId, Path pathToContainerStorage, Path pathToVespaHome, String vespaUser, String vespaUserOnHost) { @@ -71,7 +72,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override - public AthenzService identity() { + public AthenzIdentity identity() { return identity; } @@ -157,7 +158,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { public static class Builder { private NodeSpec.Builder nodeSpecBuilder = new NodeSpec.Builder(); private Acl acl; - private AthenzService identity; + private AthenzIdentity identity; private DockerNetworking dockerNetworking; private ZoneId zoneId; private Path pathToContainerStorage; @@ -192,7 +193,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } - public Builder identity(AthenzService identity) { + public Builder identity(AthenzIdentity identity) { this.identity = identity; return this; } diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index f46a2d9d2e3..2bc3ae5cfeb 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -8,12 +8,10 @@ <handler id="com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler" bundle="node-repository"> <binding>http://*/nodes/v2/*</binding> - <binding>https://*/nodes/v2/*</binding> </handler> <handler id="com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler" bundle="node-repository"> <binding>http://*/loadbalancers/v1/*</binding> - <binding>https://*/loadbalancers/v1/*</binding> </handler> <preprocess:include file="node-flavors.xml" required="false" /> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 7fdd9a168c8..3b5f9d91725 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -180,7 +180,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; DefaultTimes(Zone zone) { - failGrace = Duration.ofMinutes(60); + failGrace = Duration.ofMinutes(30); periodicRedeployInterval = Duration.ofMinutes(30); // Don't redeploy in test environments redeployMaintainerInterval = zone.environment().isTest() ? Duration.ofDays(1) : Duration.ofMinutes(1); @@ -79,6 +79,7 @@ <module>filedistribution</module> <module>flags</module> <module>fsa</module> + <module>http-utils</module> <module>indexinglanguage</module> <module>jaxrs_client_utils</module> <module>jaxrs_utils</module> diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 18d4fbdb6d7..b2334ed025e 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -5,7 +5,7 @@ LOG_SETUP("feed_and_search_test"); #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/fef.h> diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index be41aa96efd..9a1ddee4278 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/searchcorespi/index/fusionrunner.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/matchdatalayout.h> diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index d9f0a256faf..841c69289bf 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -3,28 +3,27 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> +#include <vespa/fastos/file.h> #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> +#include <vespa/searchcorespi/index/index_manager_stats.h> #include <vespa/searchcorespi/index/indexcollection.h> #include <vespa/searchcorespi/index/indexflushtarget.h> #include <vespa/searchcorespi/index/indexfusiontarget.h> -#include <vespa/searchcorespi/index/index_manager_stats.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/searchlib/common/serialnum.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/dictionary.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_index_collection.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/queryeval/isourceselector.h> -#include <vespa/searchlib/common/serialnum.h> #include <vespa/searchlib/util/dirtraverse.h> +#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/fastos/file.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <set> #include <vespa/log/log.h> @@ -43,8 +42,8 @@ using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::schema::DataType; using vespalib::makeLambdaTask; -using search::memoryindex::CompactDocumentWordsStore; -using search::memoryindex::Dictionary; +using search::memoryindex::CompactWordsStore; +using search::memoryindex::FieldIndexCollection; using search::queryeval::Source; using std::set; using std::string; @@ -359,14 +358,14 @@ TEST_F("requireThatSourceSelectorIsFlushed", Fixture) { TEST_F("requireThatFlushStatsAreCalculated", Fixture) { Schema schema(getSchema()); - Dictionary dict(schema); + FieldIndexCollection fic(schema); SequencedTaskExecutor invertThreads(2); SequencedTaskExecutor pushThreads(2); search::memoryindex::DocumentInverter inverter(schema, invertThreads, pushThreads); - uint64_t fixed_index_size = dict.getMemoryUsage().allocatedBytes(); - uint64_t index_size = dict.getMemoryUsage().allocatedBytes() - fixed_index_size; + uint64_t fixed_index_size = fic.getMemoryUsage().allocatedBytes(); + uint64_t index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. uint64_t selector_size = (1) * sizeof(Source); EXPECT_EQUAL(index_size, f._index_manager->getMaintainer().getFlushStats().memory_before_bytes - @@ -377,10 +376,10 @@ TEST_F("requireThatFlushStatsAreCalculated", Fixture) { Document::UP doc = f.addDocument(docid); inverter.invertDocument(docid, *doc); invertThreads.sync(); - inverter.pushDocuments(dict, + inverter.pushDocuments(fic, std::shared_ptr<search::IDestructorCallback>()); pushThreads.sync(); - index_size = dict.getMemoryUsage().allocatedBytes() - fixed_index_size; + index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. selector_size = (docid + 1) * sizeof(Source); @@ -397,10 +396,10 @@ TEST_F("requireThatFlushStatsAreCalculated", Fixture) { doc = f.addDocument(docid + 100); inverter.invertDocument(docid + 100, *doc); invertThreads.sync(); - inverter.pushDocuments(dict, + inverter.pushDocuments(fic, std::shared_ptr<search::IDestructorCallback>()); pushThreads.sync(); - index_size = dict.getMemoryUsage().allocatedBytes() - fixed_index_size; + index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. selector_size = (docid + 100 + 1) * sizeof(Source); EXPECT_EQUAL(index_size, diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 6124eadb34d..2ca6f969c55 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchcorespi/index/imemoryindex.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/tunefileinfo.h> diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index f03a7cfd445..66408b1d7d7 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -173,14 +173,14 @@ vespa_define_module( src/tests/index/doctypebuilder src/tests/indexmetainfo src/tests/ld-library-path - src/tests/memoryindex/compact_document_words_store + src/tests/memoryindex/compact_words_store src/tests/memoryindex/datastore - src/tests/memoryindex/dictionary - src/tests/memoryindex/document_remover - src/tests/memoryindex/documentinverter - src/tests/memoryindex/fieldinverter - src/tests/memoryindex/memoryindex - src/tests/memoryindex/urlfieldinverter + src/tests/memoryindex/document_inverter + src/tests/memoryindex/field_index + src/tests/memoryindex/field_index_remover + src/tests/memoryindex/field_inverter + src/tests/memoryindex/memory_index + src/tests/memoryindex/url_field_inverter src/tests/nativerank src/tests/nearsearch src/tests/postinglistbm diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index b8c51f4e33d..0b9cb06d2a5 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1382,7 +1382,7 @@ "public void <init>(java.util.Map)", "public void <init>(java.util.Map, java.util.Map)", "public com.yahoo.searchlib.rankingexpression.ExpressionFunction getFunction(java.lang.String)", - "protected java.util.Map functions()", + "protected final com.google.common.collect.ImmutableMap functions()", "public java.lang.String getBinding(java.lang.String)", "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index b911284a1b4..2ef8448db8b 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java index d880294660e..200e4fbe856 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java @@ -19,9 +19,9 @@ public final class FieldMatchMetrics implements Cloneable { private FieldMatchMetricsComputer source; /** The trace accumulated during execution - empty if no tracing */ - private final Trace trace=new Trace(); + private final Trace trace = new Trace(); - private boolean complete=false; + private boolean complete; // Metrics private int outOfOrder; @@ -352,7 +352,7 @@ public final class FieldMatchMetrics implements Cloneable { * </p> * * - * <p>Weight and significance are not taken into account because this is mean to capture tha quality of the + * <p>Weight and significance are not taken into account because this is meant to capture tha quality of the * match in this field, while those measures relate this match to matches in other fields. This number * can be multiplied with those values when combining with other field match scores.</p> */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java index 79886449d0a..f981ad464ec 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java @@ -136,7 +136,7 @@ public final class FieldMatchMetricsComputer { // Explore segmentations while (segmentStartPoint!=null) { - metrics =segmentStartPoint.getMetrics().clone(); + metrics = segmentStartPoint.getMetrics().clone(); if (collectTrace) metrics.trace().add("\nLooking for segment from " + segmentStartPoint + "..." + "\n"); boolean found=findAlternativeSegmentFrom(segmentStartPoint); @@ -148,7 +148,7 @@ public final class FieldMatchMetricsComputer { segmentStartPoint=findOpenSegment(segmentStartPoint.getI()); } - metrics=findLastStartPoint().getMetrics(); // these metrics are the final set + metrics = findLastStartPoint().getMetrics(); // these metrics are the final set setOccurrenceCounts(metrics); metrics.onComplete(); metrics.setComplete(true); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index ed1e2838717..084bfe65e06 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -61,7 +61,7 @@ public class FunctionReferenceContext { */ public ExpressionFunction getFunction(String name) { return functions.get(name); } - protected Map<String, ExpressionFunction> functions() { return functions; } + protected final ImmutableMap<String, ExpressionFunction> functions() { return functions; } /** Returns the resolution of an argument, or null if it isn't defined in this context */ public String getBinding(String name) { return bindings.get(name); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 0b68e71c21a..4acc1a85490 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -3,12 +3,10 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.TensorType; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -95,7 +93,7 @@ public class SerializationContext extends FunctionReferenceContext { @Override public SerializationContext withBindings(Map<String, String> bindings) { - return new SerializationContext(functions().values(), bindings, this.serializedFunctions); + return new SerializationContext(functions(), bindings, this.serializedFunctions); } public Map<String, String> serializedFunctions() { return serializedFunctions; } diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 809688bdb2e..24146e516a0 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -1,5 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/searchlib/btree/btreenode.hpp> +#include <vespa/searchlib/btree/btreenodeallocator.hpp> +#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/fusion.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/diskindex/zcposoccrandread.h> @@ -7,16 +12,11 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/btree/btreeroot.hpp> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> -#include <vespa/searchlib/btree/btreenode.hpp> -#include <vespa/searchlib/memoryindex/dictionary.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> -#include <vespa/searchlib/diskindex/diskindex.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_index_collection.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/util/filekit.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> LOG_SETUP("fusion_test"); @@ -27,7 +27,7 @@ using document::Document; using fef::FieldPositionsIterator; using fef::TermFieldMatchData; using fef::TermFieldMatchDataArray; -using memoryindex::Dictionary; +using memoryindex::FieldIndexCollection; using memoryindex::DocumentInverter; using queryeval::SearchIterator; using search::common::FileHeaderContext; @@ -54,9 +54,9 @@ public: namespace { void -myPushDocument(DocumentInverter &inv, Dictionary &d) +myPushDocument(DocumentInverter &inv, FieldIndexCollection &fieldIndexes) { - inv.pushDocuments(d, std::shared_ptr<IDestructorCallback>()); + inv.pushDocuments(fieldIndexes, std::shared_ptr<IDestructorCallback>()); } } @@ -274,7 +274,7 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, addField("f0").addField("f1"). addField("f2").addField("f3"). addField("f4")); - Dictionary d(schema); + FieldIndexCollection fic(schema); DocBuilder b(schema); SequencedTaskExecutor invertThreads(2); SequencedTaskExecutor pushThreads(2); @@ -301,7 +301,7 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, doc = b.endDocument(); inv.invertDocument(10, *doc); invertThreads.sync(); - myPushDocument(inv, d); + myPushDocument(inv, fic); pushThreads.sync(); b.startDocument("doc::11"). @@ -311,7 +311,7 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, doc = b.endDocument(); inv.invertDocument(11, *doc); invertThreads.sync(); - myPushDocument(inv, d); + myPushDocument(inv, fic); pushThreads.sync(); b.startDocument("doc::12"). @@ -321,14 +321,14 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, doc = b.endDocument(); inv.invertDocument(12, *doc); invertThreads.sync(); - myPushDocument(inv, d); + myPushDocument(inv, fic); pushThreads.sync(); IndexBuilder ib(schema); vespalib::string dump2dir = prefix + "dump2"; ib.setPrefix(dump2dir); uint32_t numDocs = 12 + 1; - uint32_t numWords = d.getNumUniqueWords(); + uint32_t numWords = fic.getNumUniqueWords(); bool dynamicKPosOcc = false; TuneFileIndexing tuneFileIndexing; TuneFileSearch tuneFileSearch; @@ -341,7 +341,7 @@ Test::requireThatFusionIsWorking(const vespalib::string &prefix, if (readmmap) tuneFileSearch._read.setWantMemoryMap(); ib.open(numDocs, numWords, tuneFileIndexing, fileHeaderContext); - d.dump(ib); + fic.dump(ib); ib.close(); vespalib::string tsName = dump2dir + "/.teststamp"; diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore deleted file mode 100644 index 3ad290f1731..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_compact_document_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt deleted file mode 100644 index a5c8e0b2d14..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_compact_document_words_store_test_app TEST - SOURCES - compact_document_words_store_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_compact_document_words_store_test_app COMMAND searchlib_compact_document_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore new file mode 100644 index 00000000000..9f9acb50adc --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore @@ -0,0 +1 @@ +searchlib_compact_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt new file mode 100644 index 00000000000..ee31ef7c7aa --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_compact_words_store_test_app TEST + SOURCES + compact_words_store_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_compact_words_store_test_app COMMAND searchlib_compact_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp index 6e22a4e5ff6..04d66396f90 100644 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp +++ b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp @@ -2,7 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/datastore/entryref.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <iostream> @@ -12,8 +12,8 @@ using namespace search; using namespace search::datastore; using namespace search::memoryindex; -typedef CompactDocumentWordsStore::Builder Builder; -typedef CompactDocumentWordsStore::Iterator Iterator; +typedef CompactWordsStore::Builder Builder; +typedef CompactWordsStore::Iterator Iterator; typedef Builder::WordRefVector WordRefVector; const EntryRef w1(1); @@ -53,7 +53,7 @@ toStr(Iterator itr) struct SingleFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; SingleFixture() : _store() { _store.insert(Builder(d1).insert(w1).insert(w2).insert(w3)); } @@ -61,7 +61,7 @@ struct SingleFixture struct MultiFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; MultiFixture() : _store() { _store.insert(Builder(d1).insert(w1)); _store.insert(Builder(d2).insert(w2)); @@ -100,7 +100,7 @@ TEST_F("require that documents can be removed and re-inserted", MultiFixture) TEST("require that a lot of words can be inserted, retrieved and removed") { - CompactDocumentWordsStore store; + CompactWordsStore store; for (uint32_t docId = 0; docId < 50; ++docId) { Builder b(docId); for (uint32_t wordRef = 0; wordRef < 20000; ++wordRef) { @@ -125,9 +125,9 @@ TEST("require that a lot of words can be inserted, retrieved and removed") TEST("require that initial memory usage is reported") { - CompactDocumentWordsStore store; - CompactDocumentWordsStore::DocumentWordsMap docs; - CompactDocumentWordsStore::Store internalStore; + CompactWordsStore store; + CompactWordsStore::DocumentWordsMap docs; + CompactWordsStore::Store internalStore; MemoryUsage initExp; initExp.incAllocatedBytes(docs.getMemoryConsumption()); initExp.incUsedBytes(docs.getMemoryUsed()); @@ -142,7 +142,7 @@ TEST("require that initial memory usage is reported") TEST("require that memory usage is updated after insert") { - CompactDocumentWordsStore store; + CompactWordsStore store; MemoryUsage init = store.getMemoryUsage(); store.insert(Builder(d1).insert(w1)); diff --git a/searchlib/src/tests/memoryindex/datastore/.gitignore b/searchlib/src/tests/memoryindex/datastore/.gitignore index 98f4acc70a8..a5842a0fd69 100644 --- a/searchlib/src/tests/memoryindex/datastore/.gitignore +++ b/searchlib/src/tests/memoryindex/datastore/.gitignore @@ -1,8 +1,4 @@ .depend Makefile -datastore_test -featurestore_test -wordstore_test -searchlib_datastore_test_app -searchlib_featurestore_test_app -searchlib_wordstore_test_app +searchlib_feature_store_test_app +searchlib_word_store_test_app diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt index 2ba0f2eac63..45507f3b0ae 100644 --- a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -1,15 +1,15 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_featurestore_test_app TEST +vespa_add_executable(searchlib_feature_store_test_app TEST SOURCES - featurestore_test.cpp + feature_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_featurestore_test_app COMMAND searchlib_featurestore_test_app) -vespa_add_executable(searchlib_wordstore_test_app TEST +vespa_add_test(NAME searchlib_feature_store_test_app COMMAND searchlib_feature_store_test_app) +vespa_add_executable(searchlib_word_store_test_app TEST SOURCES - wordstore_test.cpp + word_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_wordstore_test_app COMMAND searchlib_wordstore_test_app) +vespa_add_test(NAME searchlib_word_store_test_app COMMAND searchlib_word_store_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index dc061f55732..49e9d613861 100644 --- a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("featurestore_test"); +LOG_SETUP("feature_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> using namespace search::btree; using namespace search::datastore; @@ -213,7 +213,7 @@ Test::Test() int Test::Main() { - TEST_INIT("featurestore_test"); + TEST_INIT("feature_store_test"); requireThatFeaturesCanBeAddedAndRetrieved(); requireThatNextWordsAreWorking(); diff --git a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp index c1baff72514..b7f454bfdf7 100644 --- a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("wordstore_test"); +LOG_SETUP("word_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> using namespace search::datastore; @@ -75,7 +75,7 @@ Test::requireThatAddWordTriggersChangeOfBuffer() int Test::Main() { - TEST_INIT("wordstore_test"); + TEST_INIT("word_store_test"); requireThatWordsCanBeAddedAndRetrieved(); requireThatAddWordTriggersChangeOfBuffer(); diff --git a/searchlib/src/tests/memoryindex/dictionary/.gitignore b/searchlib/src/tests/memoryindex/dictionary/.gitignore deleted file mode 100644 index d404d7d7063..00000000000 --- a/searchlib/src/tests/memoryindex/dictionary/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -.depend -Makefile -dictionary_test -dump -/urldump -searchlib_dictionary_test_app diff --git a/searchlib/src/tests/memoryindex/document_inverter/.gitignore b/searchlib/src/tests/memoryindex/document_inverter/.gitignore new file mode 100644 index 00000000000..6245bb146e7 --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_document_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt new file mode 100644 index 00000000000..1058a19d0ce --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_document_inverter_test_app TEST + SOURCES + document_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_document_inverter_test_app COMMAND searchlib_document_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index 36cd15c8ada..91c1ccba706 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -3,9 +3,9 @@ #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> @@ -97,7 +97,7 @@ struct Fixture SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; DocumentInverter _inv; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt deleted file mode 100644 index 04e5b0ec126..00000000000 --- a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_document_remover_test_app TEST - SOURCES - document_remover_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_document_remover_test_app COMMAND searchlib_document_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/.gitignore b/searchlib/src/tests/memoryindex/documentinverter/.gitignore deleted file mode 100644 index 1e9666b2d63..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_documentinverter_test_app diff --git a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt deleted file mode 100644 index c7760e9f12f..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_documentinverter_test_app TEST - SOURCES - documentinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_documentinverter_test_app COMMAND searchlib_documentinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/.gitignore b/searchlib/src/tests/memoryindex/field_index/.gitignore new file mode 100644 index 00000000000..e1a6c049431 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_index/.gitignore @@ -0,0 +1,6 @@ +.depend +Makefile +field_index_test +dump +/urldump +searchlib_field_index_test_app diff --git a/searchlib/src/tests/memoryindex/dictionary/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt index 5866e1d3413..767097b99db 100644 --- a/searchlib/src/tests/memoryindex/dictionary/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt @@ -1,9 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_dictionary_test_app TEST +vespa_add_executable(searchlib_field_index_test_app TEST SOURCES - dictionary_test.cpp + field_index_test.cpp DEPENDS searchlib searchlib_test ) -vespa_add_test(NAME searchlib_dictionary_test_app COMMAND searchlib_dictionary_test_app) +vespa_add_test(NAME searchlib_field_index_test_app COMMAND searchlib_field_index_test_app) diff --git a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 2c4fdc35ac6..3a635756ec7 100644 --- a/searchlib/src/tests/memoryindex/dictionary/dictionary_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -1,5 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/searchlib/btree/btreenodeallocator.hpp> +#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/diskindex/fusion.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/diskindex/zcposoccrandread.h> @@ -7,14 +10,11 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/btree/btreeroot.hpp> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> -#include <vespa/searchlib/memoryindex/dictionary.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_index_collection.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/vespalib/testkit/testapp.h> @@ -37,7 +37,7 @@ using vespalib::GenerationHandler; namespace memoryindex { -typedef Dictionary::PostingList PostingList; +typedef FieldIndex::PostingList PostingList; typedef PostingList::ConstIterator PostingConstItr; class MyBuilder : public IndexBuilder { @@ -240,17 +240,17 @@ namespace { /** - * MockDictionary is a simple mockup of memory index, used to verify - * that we get correct posting lists from real memory index. + * A simple mockup of a memory field index, used to verify + * that we get correct posting lists from real memory field index. */ -class MockDictionary +class MockFieldIndex { std::map<std::pair<vespalib::string, uint32_t>, std::set<uint32_t>> _dict; vespalib::string _word; uint32_t _fieldId; public: - ~MockDictionary(); + ~MockFieldIndex(); void setNextWord(const vespalib::string &word) { @@ -296,11 +296,11 @@ public: } }; -MockDictionary::~MockDictionary() = default; +MockFieldIndex::~MockFieldIndex() = default; /** * MockWordStoreScan is a helper class to ensure that previous word is - * still stored safely in memory, to satisfy OrderedDocumentInserter + * still stored safely in memory, to satisfy OrderedFieldIndexInserter * needs. */ class MockWordStoreScan @@ -335,6 +335,7 @@ public: }; MockWordStoreScan::~MockWordStoreScan() = default; + /** * MyInserter performs insertions on both a mockup version of memory index * and a real memory index. Mockup version is used to calculate expected @@ -343,18 +344,18 @@ MockWordStoreScan::~MockWordStoreScan() = default; class MyInserter { MockWordStoreScan _wordStoreScan; - MockDictionary _mock; - Dictionary _d; + MockFieldIndex _mock; + FieldIndexCollection _fieldIndexes; DocIdAndPosOccFeatures _features; - IOrderedDocumentInserter *_documentInserter; + IOrderedFieldIndexInserter *_inserter; public: MyInserter(const Schema &schema) : _wordStoreScan(), _mock(), - _d(schema), + _fieldIndexes(schema), _features(), - _documentInserter(nullptr) + _inserter(nullptr) { _features.addNextOcc(0, 0, 1, 1); } @@ -364,32 +365,32 @@ public: setNextWord(const vespalib::string &word) { const vespalib::string &w = _wordStoreScan.setWord(word); - _documentInserter->setNextWord(w); + _inserter->setNextWord(w); _mock.setNextWord(w); } void setNextField(uint32_t fieldId) { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } - _documentInserter = &_d.getFieldIndex(fieldId)->getInserter(); - _documentInserter->rewind(); + _inserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); + _inserter->rewind(); _mock.setNextField(fieldId); } void add(uint32_t docId) { - _documentInserter->add(docId, _features); + _inserter->add(docId, _features); _mock.add(docId); } void remove(uint32_t docId) { - _documentInserter->remove(docId); + _inserter->remove(docId); _mock.remove(docId); } @@ -398,15 +399,15 @@ public: uint32_t fieldId) { std::vector<uint32_t> exp = _mock.find(word, fieldId); - PostingConstItr itr = _d.find(word, fieldId); + PostingConstItr itr = _fieldIndexes.find(word, fieldId); return EXPECT_TRUE(assertPostingList(exp, itr)); } bool assertPostings() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } for (auto wfp : _mock) { auto &wf = wfp.first; @@ -422,38 +423,38 @@ public: void rewind() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); - _documentInserter = nullptr; + if (_inserter != nullptr) { + _inserter->flush(); + _inserter = nullptr; } } uint32_t getNumUniqueWords() { - return _d.getNumUniqueWords(); + return _fieldIndexes.getNumUniqueWords(); } - Dictionary &getDict() { return _d; } + FieldIndexCollection &getFieldIndexes() { return _fieldIndexes; } }; MyInserter::~MyInserter() = default; void -myremove(uint32_t docId, DocumentInverter &inv, Dictionary &d, +myremove(uint32_t docId, DocumentInverter &inv, FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &invertThreads) { inv.removeDocument(docId); invertThreads.sync(); - inv.pushDocuments(d, std::shared_ptr<IDestructorCallback>()); + inv.pushDocuments(fieldIndexes, std::shared_ptr<IDestructorCallback>()); } class WrapInserter { - OrderedDocumentInserter &_inserter; + OrderedFieldIndexInserter &_inserter; public: - WrapInserter(Dictionary &d, uint32_t fieldId) - : _inserter(d.getFieldIndex(fieldId)->getInserter()) + WrapInserter(FieldIndexCollection &fieldIndexes, uint32_t fieldId) + : _inserter(fieldIndexes.getFieldIndex(fieldId)->getInserter()) { } @@ -502,14 +503,14 @@ public: }; -class MyDrainRemoves : IDocumentRemoveListener +class MyDrainRemoves : IFieldIndexRemoveListener { - DocumentRemover &_remover; + FieldIndexRemover &_remover; public: virtual void remove(const vespalib::stringref, uint32_t) override { } - MyDrainRemoves(Dictionary &d, uint32_t fieldId) - : _remover(d.getFieldIndex(fieldId)->getDocumentRemover()) + MyDrainRemoves(FieldIndexCollection &fieldIndexes, uint32_t fieldId) + : _remover(fieldIndexes.getFieldIndex(fieldId)->getDocumentRemover()) { } @@ -520,43 +521,43 @@ public: }; void -myPushDocument(DocumentInverter &inv, Dictionary &d) +myPushDocument(DocumentInverter &inv, FieldIndexCollection &fieldIndexes) { - inv.pushDocuments(d, std::shared_ptr<IDestructorCallback>()); + inv.pushDocuments(fieldIndexes, std::shared_ptr<IDestructorCallback>()); } const FeatureStore * -featureStorePtr(const Dictionary &d, uint32_t fieldId) +featureStorePtr(const FieldIndexCollection &fieldIndexes, uint32_t fieldId) { - return &d.getFieldIndex(fieldId)->getFeatureStore(); + return &fieldIndexes.getFieldIndex(fieldId)->getFeatureStore(); } const FeatureStore & -featureStoreRef(const Dictionary &d, uint32_t fieldId) +featureStoreRef(const FieldIndexCollection &fieldIndexes, uint32_t fieldId) { - return d.getFieldIndex(fieldId)->getFeatureStore(); + return fieldIndexes.getFieldIndex(fieldId)->getFeatureStore(); } DataStoreBase::MemStats -getFeatureStoreMemStats(const Dictionary &d) +getFeatureStoreMemStats(const FieldIndexCollection &fieldIndexes) { DataStoreBase::MemStats res; - uint32_t numFields = d.getNumFields(); + uint32_t numFields = fieldIndexes.getNumFields(); for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { DataStoreBase::MemStats stats = - d.getFieldIndex(fieldId)->getFeatureStore().getMemStats(); + fieldIndexes.getFieldIndex(fieldId)->getFeatureStore().getMemStats(); res += stats; } return res; } -void myCommit(Dictionary &d, ISequencedTaskExecutor &pushThreads) +void myCommit(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pushThreads) { uint32_t fieldId = 0; - for (auto &fieldIndex : d.getFieldIndexes()) { + for (auto &fieldIndex : fieldIndexes.getFieldIndexes()) { pushThreads.execute(fieldId, [fieldIndex(fieldIndex.get())]() { fieldIndex->commit(); }); @@ -567,10 +568,10 @@ void myCommit(Dictionary &d, ISequencedTaskExecutor &pushThreads) void -myCompactFeatures(Dictionary &d, ISequencedTaskExecutor &pushThreads) +myCompactFeatures(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pushThreads) { uint32_t fieldId = 0; - for (auto &fieldIndex : d.getFieldIndexes()) { + for (auto &fieldIndex : fieldIndexes.getFieldIndexes()) { pushThreads.execute(fieldId, [fieldIndex(fieldIndex.get())]() { fieldIndex->compactFeatures(); }); @@ -593,67 +594,69 @@ struct Fixture const Schema & getSchema() const { return _schema; } }; +// TODO: Rewrite most tests to use FieldIndex directly instead of going via FieldIndexCollection. + TEST_F("requireThatFreshInsertWorks", Fixture) { - Dictionary d(f.getSchema()); + FieldIndexCollection fic(f.getSchema()); SequencedTaskExecutor pushThreads(2); - EXPECT_TRUE(assertPostingList("[]", d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", d.findFrozen("a", 0))); - EXPECT_EQUAL(0u, d.getNumUniqueWords()); - WrapInserter(d, 0).word("a").add(10).flush(); - EXPECT_TRUE(assertPostingList("[10]", d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", d.findFrozen("a", 0))); - myCommit(d, pushThreads); - EXPECT_TRUE(assertPostingList("[10]", d.findFrozen("a", 0))); - EXPECT_EQUAL(1u, d.getNumUniqueWords()); + EXPECT_TRUE(assertPostingList("[]", fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); + EXPECT_EQUAL(0u, fic.getNumUniqueWords()); + WrapInserter(fic, 0).word("a").add(10).flush(); + EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); + myCommit(fic, pushThreads); + EXPECT_TRUE(assertPostingList("[10]", fic.findFrozen("a", 0))); + EXPECT_EQUAL(1u, fic.getNumUniqueWords()); } TEST_F("requireThatAppendInsertWorks", Fixture) { - Dictionary d(f.getSchema()); + FieldIndexCollection fic(f.getSchema()); SequencedTaskExecutor pushThreads(2); - WrapInserter(d, 0).word("a").add(10).flush().rewind(). + WrapInserter(fic, 0).word("a").add(10).flush().rewind(). word("a").add(5).flush(); - EXPECT_TRUE(assertPostingList("[5,10]", d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", d.findFrozen("a", 0))); - WrapInserter(d, 0).rewind().word("a").add(20).flush(); - EXPECT_TRUE(assertPostingList("[5,10,20]", d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", d.findFrozen("a", 0))); - myCommit(d, pushThreads); - EXPECT_TRUE(assertPostingList("[5,10,20]", d.findFrozen("a", 0))); + EXPECT_TRUE(assertPostingList("[5,10]", fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); + WrapInserter(fic, 0).rewind().word("a").add(20).flush(); + EXPECT_TRUE(assertPostingList("[5,10,20]", fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); + myCommit(fic, pushThreads); + EXPECT_TRUE(assertPostingList("[5,10,20]", fic.findFrozen("a", 0))); } TEST_F("requireThatMultiplePostingListsCanExist", Fixture) { - Dictionary d(f.getSchema()); - WrapInserter(d, 0).word("a").add(10).word("b").add(11).add(15).flush(); - WrapInserter(d, 1).word("a").add(5).word("b").add(12).flush(); - EXPECT_EQUAL(4u, d.getNumUniqueWords()); - EXPECT_TRUE(assertPostingList("[10]", d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[5]", d.find("a", 1))); - EXPECT_TRUE(assertPostingList("[11,15]", d.find("b", 0))); - EXPECT_TRUE(assertPostingList("[12]", d.find("b", 1))); - EXPECT_TRUE(assertPostingList("[]", d.find("a", 2))); - EXPECT_TRUE(assertPostingList("[]", d.find("c", 0))); + FieldIndexCollection fic(f.getSchema()); + WrapInserter(fic, 0).word("a").add(10).word("b").add(11).add(15).flush(); + WrapInserter(fic, 1).word("a").add(5).word("b").add(12).flush(); + EXPECT_EQUAL(4u, fic.getNumUniqueWords()); + EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[5]", fic.find("a", 1))); + EXPECT_TRUE(assertPostingList("[11,15]", fic.find("b", 0))); + EXPECT_TRUE(assertPostingList("[12]", fic.find("b", 1))); + EXPECT_TRUE(assertPostingList("[]", fic.find("a", 2))); + EXPECT_TRUE(assertPostingList("[]", fic.find("c", 0))); } TEST_F("requireThatRemoveWorks", Fixture) { - Dictionary d(f.getSchema()); - WrapInserter(d, 0).word("a").remove(10).flush(); - EXPECT_TRUE(assertPostingList("[]", d.find("a", 0))); - WrapInserter(d, 0).add(10).add(20).add(30).flush(); - EXPECT_TRUE(assertPostingList("[10,20,30]", d.find("a", 0))); - WrapInserter(d, 0).rewind().word("a").remove(10).flush(); - EXPECT_TRUE(assertPostingList("[20,30]", d.find("a", 0))); - WrapInserter(d, 0).remove(20).flush(); - EXPECT_TRUE(assertPostingList("[30]", d.find("a", 0))); - WrapInserter(d, 0).remove(30).flush(); - EXPECT_TRUE(assertPostingList("[]", d.find("a", 0))); - EXPECT_EQUAL(1u, d.getNumUniqueWords()); - MyDrainRemoves(d, 0).drain(10); - WrapInserter(d, 0).rewind().word("a").add(10).flush(); - EXPECT_TRUE(assertPostingList("[10]", d.find("a", 0))); + FieldIndexCollection fic(f.getSchema()); + WrapInserter(fic, 0).word("a").remove(10).flush(); + EXPECT_TRUE(assertPostingList("[]", fic.find("a", 0))); + WrapInserter(fic, 0).add(10).add(20).add(30).flush(); + EXPECT_TRUE(assertPostingList("[10,20,30]", fic.find("a", 0))); + WrapInserter(fic, 0).rewind().word("a").remove(10).flush(); + EXPECT_TRUE(assertPostingList("[20,30]", fic.find("a", 0))); + WrapInserter(fic, 0).remove(20).flush(); + EXPECT_TRUE(assertPostingList("[30]", fic.find("a", 0))); + WrapInserter(fic, 0).remove(30).flush(); + EXPECT_TRUE(assertPostingList("[]", fic.find("a", 0))); + EXPECT_EQUAL(1u, fic.getNumUniqueWords()); + MyDrainRemoves(fic, 0).drain(10); + WrapInserter(fic, 0).rewind().word("a").add(10).flush(); + EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); } TEST_F("requireThatMultipleInsertAndRemoveWorks", Fixture) @@ -675,7 +678,7 @@ TEST_F("requireThatMultipleInsertAndRemoveWorks", Fixture) EXPECT_TRUE(inserter.assertPostings()); inserter.rewind(); for (uint32_t fi = 0; fi < numFields; ++fi) { - MyDrainRemoves drainRemoves(inserter.getDict(), fi); + MyDrainRemoves drainRemoves(inserter.getFieldIndexes(), fi); for (uint32_t di = 0; di < 'z' * 2 + 1; ++di) { drainRemoves.drain(di); } @@ -723,20 +726,20 @@ getFeatures(uint32_t elemLen, uint32_t numOccs, int32_t weight = 1) TEST_F("requireThatFeaturesAreInPostingLists", Fixture) { - Dictionary d(f.getSchema()); - WrapInserter(d, 0).word("a").add(1, getFeatures(4, 2)).flush(); + FieldIndexCollection fic(f.getSchema()); + WrapInserter(fic, 0).word("a").add(1, getFeatures(4, 2)).flush(); EXPECT_TRUE(assertPostingList("[1{4:0,1}]", - d.find("a", 0), - featureStorePtr(d, 0))); - WrapInserter(d, 0).word("b").add(2, getFeatures(5, 1)). + fic.find("a", 0), + featureStorePtr(fic, 0))); + WrapInserter(fic, 0).word("b").add(2, getFeatures(5, 1)). add(3, getFeatures(6, 2)).flush(); EXPECT_TRUE(assertPostingList("[2{5:0},3{6:0,1}]", - d.find("b", 0), - featureStorePtr(d, 0))); - WrapInserter(d, 1).word("c").add(4, getFeatures(7, 2)).flush(); + fic.find("b", 0), + featureStorePtr(fic, 0))); + WrapInserter(fic, 1).word("c").add(4, getFeatures(7, 2)).flush(); EXPECT_TRUE(assertPostingList("[4{7:0,1}]", - d.find("c", 1), - featureStorePtr(d, 1))); + fic.find("c", 1), + featureStorePtr(fic, 1))); } class Verifier : public SearchIteratorVerifier { @@ -748,20 +751,20 @@ public: (void) strict; TermFieldMatchDataArray matchData; matchData.add(&_tfmd); - return std::make_unique<PostingIterator>(_dictionary.find("a", 0), featureStoreRef(_dictionary, 0), 0, matchData); + return std::make_unique<PostingIterator>(_fieldIndexes.find("a", 0), featureStoreRef(_fieldIndexes, 0), 0, matchData); } private: mutable TermFieldMatchData _tfmd; - Dictionary _dictionary; + FieldIndexCollection _fieldIndexes; }; Verifier::Verifier(const Schema & schema) : _tfmd(), - _dictionary(schema) + _fieldIndexes(schema) { - WrapInserter inserter(_dictionary, 0); + WrapInserter inserter(_fieldIndexes, 0); inserter.word("a"); for (uint32_t docId : getExpectedDocIds()) { inserter.add(docId); @@ -778,8 +781,8 @@ TEST_F("require that postingiterator conforms", Fixture) { TEST_F("requireThatPostingIteratorIsWorking", Fixture) { - Dictionary d(f.getSchema()); - WrapInserter(d, 0).word("a").add(10, getFeatures(4, 1)). + FieldIndexCollection fic(f.getSchema()); + WrapInserter(fic, 0).word("a").add(10, getFeatures(4, 1)). add(20, getFeatures(5, 2)). add(30, getFeatures(6, 1)). add(40, getFeatures(7, 2)).flush(); @@ -787,15 +790,15 @@ TEST_F("requireThatPostingIteratorIsWorking", Fixture) TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - PostingIterator itr(d.find("not", 0), - featureStoreRef(d, 0), + PostingIterator itr(fic.find("not", 0), + featureStoreRef(fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(d.find("a", 0), - featureStoreRef(d, 0), + PostingIterator itr(fic.find("a", 0), + featureStoreRef(fic, 0), 0, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); @@ -834,28 +837,28 @@ TEST_F("requireThatDumpingToIndexBuilderIsWorking", Fixture) EXPECT_EQUAL("f=4[w=a[d=2[e=0,w=10,l=20[1,3]]]]", b.toStr()); } { - Dictionary d(f.getSchema()); + FieldIndexCollection fic(f.getSchema()); MyBuilder b(f.getSchema()); DocIdAndFeatures df; - WrapInserter(d, 1).word("a").add(5, getFeatures(2, 1)). + WrapInserter(fic, 1).word("a").add(5, getFeatures(2, 1)). add(7, getFeatures(3, 2)). word("b").add(5, getFeatures(12, 2)).flush(); df = getFeatures(4, 1); addElement(df, 5, 2); - WrapInserter(d, 2).word("a").add(5, df); + WrapInserter(fic, 2).word("a").add(5, df); df = getFeatures(6, 1); addElement(df, 7, 2); - WrapInserter(d, 2).add(7, df).flush(); + WrapInserter(fic, 2).add(7, df).flush(); df = getFeatures(8, 1, 12); addElement(df, 9, 2, 13); - WrapInserter(d, 3).word("a").add(5, df); + WrapInserter(fic, 3).word("a").add(5, df); df = getFeatures(10, 1, 14); addElement(df, 11, 2, 15); - WrapInserter(d, 3).add(7, df).flush(); + WrapInserter(fic, 3).add(7, df).flush(); - d.dump(b); + fic.dump(b); EXPECT_EQUAL("f=0[]," "f=1[w=a[d=5[e=0,w=1,l=2[0]],d=7[e=0,w=1,l=3[0,1]]]," @@ -867,13 +870,13 @@ TEST_F("requireThatDumpingToIndexBuilderIsWorking", Fixture) b.toStr()); } { // test word with no docs - Dictionary d(f.getSchema()); - WrapInserter(d, 0).word("a").add(2, getFeatures(2, 1)). + FieldIndexCollection fic(f.getSchema()); + WrapInserter(fic, 0).word("a").add(2, getFeatures(2, 1)). word("b").add(4, getFeatures(4, 1)).flush().rewind(). word("a").remove(2).flush(); { MyBuilder b(f.getSchema()); - d.dump(b); + fic.dump(b); EXPECT_EQUAL("f=0[w=b[d=4[e=0,w=1,l=4[0]]]],f=1[],f=2[],f=3[]", b.toStr()); } @@ -883,7 +886,7 @@ TEST_F("requireThatDumpingToIndexBuilderIsWorking", Fixture) TuneFileIndexing tuneFileIndexing; DummyFileHeaderContext fileHeaderContext; b.open(5, 2, tuneFileIndexing, fileHeaderContext); - d.dump(b); + fic.dump(b); b.close(); } } @@ -891,19 +894,19 @@ TEST_F("requireThatDumpingToIndexBuilderIsWorking", Fixture) template <typename FixtureBase> -class DictionaryFixture : public FixtureBase +class FieldIndexFixture : public FixtureBase { public: using FixtureBase::getSchema; - Dictionary _d; + FieldIndexCollection _fic; DocBuilder _b; SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; DocumentInverter _inv; - DictionaryFixture() + FieldIndexFixture() : FixtureBase(), - _d(getSchema()), + _fic(getSchema()), _b(getSchema()), _invertThreads(2), _pushThreads(2), @@ -913,7 +916,7 @@ public: }; -TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) +TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) { Document::UP doc; @@ -924,7 +927,7 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) doc = f._b.endDocument(); f._inv.invertDocument(10, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); f._b.startDocument("doc::20"); @@ -934,7 +937,7 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) doc = f._b.endDocument(); f._inv.invertDocument(20, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); f._b.startDocument("doc::30"); @@ -965,7 +968,7 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) doc = f._b.endDocument(); f._inv.invertDocument(30, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); f._b.startDocument("doc::40"); @@ -976,7 +979,7 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) doc = f._b.endDocument(); f._inv.invertDocument(40, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); f._b.startDocument("doc::999"); @@ -1006,12 +1009,12 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) for (uint32_t docId = 10000; docId < 20000; ++docId) { f._inv.invertDocument(docId, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); } f._pushThreads.sync(); - DataStoreBase::MemStats beforeStats = getFeatureStoreMemStats(f._d); + DataStoreBase::MemStats beforeStats = getFeatureStoreMemStats(f._fic); LOG(info, "Before feature compaction: allocElems=%zu, usedElems=%zu" ", deadElems=%zu, holdElems=%zu" @@ -1024,14 +1027,14 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) beforeStats._freeBuffers, beforeStats._activeBuffers, beforeStats._holdBuffers); - myCompactFeatures(f._d, f._pushThreads); + myCompactFeatures(f._fic, f._pushThreads); std::vector<std::unique_ptr<GenerationHandler::Guard>> guards; - for (auto &fieldIndex : f._d.getFieldIndexes()) { + for (auto &fieldIndex : f._fic.getFieldIndexes()) { guards.push_back(std::make_unique<GenerationHandler::Guard> (fieldIndex->takeGenerationGuard())); } - myCommit(f._d, f._pushThreads); - DataStoreBase::MemStats duringStats = getFeatureStoreMemStats(f._d); + myCommit(f._fic, f._pushThreads); + DataStoreBase::MemStats duringStats = getFeatureStoreMemStats(f._fic); LOG(info, "During feature compaction: allocElems=%zu, usedElems=%zu" ", deadElems=%zu, holdElems=%zu" @@ -1045,8 +1048,8 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) duringStats._activeBuffers, duringStats._holdBuffers); guards.clear(); - myCommit(f._d, f._pushThreads); - DataStoreBase::MemStats afterStats = getFeatureStoreMemStats(f._d); + myCommit(f._fic, f._pushThreads); + DataStoreBase::MemStats afterStats = getFeatureStoreMemStats(f._fic); LOG(info, "After feature compaction: allocElems=%zu, usedElems=%zu" ", deadElems=%zu, holdElems=%zu" @@ -1064,12 +1067,12 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - PostingIterator itr(f._d.findFrozen("not", 0), featureStoreRef(f._d, 0), 0, matchData); + PostingIterator itr(f._fic.findFrozen("not", 0), featureStoreRef(f._fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(f._d.findFrozen("a", 0), featureStoreRef(f._d, 0), 0, matchData); + PostingIterator itr(f._fic.findFrozen("a", 0), featureStoreRef(f._fic, 0), 0, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); itr.unpack(10); @@ -1086,19 +1089,19 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(f._d.findFrozen("x", 0), featureStoreRef(f._d, 0), 0, matchData); + PostingIterator itr(f._fic.findFrozen("x", 0), featureStoreRef(f._fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(f._d.findFrozen("x", 1), featureStoreRef(f._d, 1), 1, matchData); + PostingIterator itr(f._fic.findFrozen("x", 1), featureStoreRef(f._fic, 1), 1, matchData); itr.initFullRange(); EXPECT_EQUAL(30u, itr.getDocId()); itr.unpack(30); EXPECT_EQUAL("{6:2[e=0,w=1,l=6]}", toString(tfmd.getIterator(), true, true)); } { - PostingIterator itr(f._d.findFrozen("x", 2), featureStoreRef(f._d, 2), 2, matchData); + PostingIterator itr(f._fic.findFrozen("x", 2), featureStoreRef(f._fic, 2), 2, matchData); itr.initFullRange(); EXPECT_EQUAL(30u, itr.getDocId()); itr.unpack(30); @@ -1106,7 +1109,7 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) EXPECT_EQUAL("{2:1[e=0,w=1,l=2]}", toString(tfmd.getIterator(), true, true)); } { - PostingIterator itr(f._d.findFrozen("x", 3), featureStoreRef(f._d, 3), 3, matchData); + PostingIterator itr(f._fic.findFrozen("x", 3), featureStoreRef(f._fic, 3), 3, matchData); itr.initFullRange(); EXPECT_EQUAL(30u, itr.getDocId()); itr.unpack(30); @@ -1116,7 +1119,7 @@ TEST_F("requireThatInversionIsWorking", DictionaryFixture<Fixture>) } TEST_F("requireThatInverterHandlesRemoveViaDocumentRemover", - DictionaryFixture<Fixture>) + FieldIndexFixture<Fixture>) { Document::UP doc; @@ -1126,7 +1129,7 @@ TEST_F("requireThatInverterHandlesRemoveViaDocumentRemover", Document::UP doc1 = f._b.endDocument(); f._inv.invertDocument(1, *doc1.get()); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); f._b.startDocument("doc::2"); @@ -1134,23 +1137,23 @@ TEST_F("requireThatInverterHandlesRemoveViaDocumentRemover", Document::UP doc2 = f._b.endDocument(); f._inv.invertDocument(2, *doc2.get()); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); - EXPECT_TRUE(assertPostingList("[1]", f._d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[1,2]", f._d.find("b", 0))); - EXPECT_TRUE(assertPostingList("[2]", f._d.find("c", 0))); - EXPECT_TRUE(assertPostingList("[1]", f._d.find("a", 1))); - EXPECT_TRUE(assertPostingList("[1]", f._d.find("c", 1))); + EXPECT_TRUE(assertPostingList("[1]", f._fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[1,2]", f._fic.find("b", 0))); + EXPECT_TRUE(assertPostingList("[2]", f._fic.find("c", 0))); + EXPECT_TRUE(assertPostingList("[1]", f._fic.find("a", 1))); + EXPECT_TRUE(assertPostingList("[1]", f._fic.find("c", 1))); - myremove(1, f._inv, f._d, f._invertThreads); + myremove(1, f._inv, f._fic, f._invertThreads); f._pushThreads.sync(); - EXPECT_TRUE(assertPostingList("[]", f._d.find("a", 0))); - EXPECT_TRUE(assertPostingList("[2]", f._d.find("b", 0))); - EXPECT_TRUE(assertPostingList("[2]", f._d.find("c", 0))); - EXPECT_TRUE(assertPostingList("[]", f._d.find("a", 1))); - EXPECT_TRUE(assertPostingList("[]", f._d.find("c", 1))); + EXPECT_TRUE(assertPostingList("[]", f._fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[2]", f._fic.find("b", 0))); + EXPECT_TRUE(assertPostingList("[2]", f._fic.find("c", 0))); + EXPECT_TRUE(assertPostingList("[]", f._fic.find("a", 1))); + EXPECT_TRUE(assertPostingList("[]", f._fic.find("c", 1))); } class UriFixture @@ -1168,7 +1171,7 @@ public: }; -TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) +TEST_F("requireThatUriIndexingIsWorking", FieldIndexFixture<UriFixture>) { Document::UP doc; @@ -1295,7 +1298,7 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) doc = f._b.endDocument(); f._inv.invertDocument(10, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); @@ -1304,16 +1307,16 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) matchData.add(&tfmd); { uint32_t fieldId = f.getSchema().getIndexFieldId("iu"); - PostingIterator itr(f._d.findFrozen("not", fieldId), - featureStoreRef(f._d, fieldId), + PostingIterator itr(f._fic.findFrozen("not", fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { uint32_t fieldId = f.getSchema().getIndexFieldId("iu"); - PostingIterator itr(f._d.findFrozen("example", fieldId), - featureStoreRef(f._d, fieldId), + PostingIterator itr(f._fic.findFrozen("example", fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); @@ -1324,8 +1327,8 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) } { uint32_t fieldId = f.getSchema().getIndexFieldId("iau"); - PostingIterator itr(f._d.findFrozen("example", fieldId), - featureStoreRef(f._d, fieldId), + PostingIterator itr(f._fic.findFrozen("example", fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); @@ -1337,8 +1340,8 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) } { uint32_t fieldId = f.getSchema().getIndexFieldId("iwu"); - PostingIterator itr(f._d.findFrozen("example", fieldId), - featureStoreRef(f._d, fieldId), + PostingIterator itr(f._fic.findFrozen("example", fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); @@ -1353,9 +1356,9 @@ TEST_F("requireThatUriIndexingIsWorking", DictionaryFixture<UriFixture>) dib.setPrefix("urldump"); TuneFileIndexing tuneFileIndexing; DummyFileHeaderContext fileHeaderContext; - dib.open(11, f._d.getNumUniqueWords(), tuneFileIndexing, + dib.open(11, f._fic.getNumUniqueWords(), tuneFileIndexing, fileHeaderContext); - f._d.dump(dib); + f._fic.dump(dib); dib.close(); } } @@ -1373,7 +1376,7 @@ public: const Schema & getSchema() const { return _schema; } }; -TEST_F("requireThatCjkIndexingIsWorking", DictionaryFixture<SingleFieldFixture>) +TEST_F("requireThatCjkIndexingIsWorking", FieldIndexFixture<SingleFieldFixture>) { Document::UP doc; @@ -1387,7 +1390,7 @@ TEST_F("requireThatCjkIndexingIsWorking", DictionaryFixture<SingleFieldFixture>) doc = f._b.endDocument(); f._inv.invertDocument(10, *doc); f._invertThreads.sync(); - myPushDocument(f._inv, f._d); + myPushDocument(f._inv, f._fic); f._pushThreads.sync(); @@ -1396,18 +1399,18 @@ TEST_F("requireThatCjkIndexingIsWorking", DictionaryFixture<SingleFieldFixture>) matchData.add(&tfmd); { uint32_t fieldId = f.getSchema().getIndexFieldId("i"); - PostingIterator itr(f._d.findFrozen("not", fieldId), - featureStoreRef(f._d, fieldId), + PostingIterator itr(f._fic.findFrozen("not", fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { uint32_t fieldId = f.getSchema().getIndexFieldId("i"); - PostingIterator itr(f._d.findFrozen("我就" + PostingIterator itr(f._fic.findFrozen("我就" "是那个", fieldId), - featureStoreRef(f._d, fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); @@ -1418,10 +1421,10 @@ TEST_F("requireThatCjkIndexingIsWorking", DictionaryFixture<SingleFieldFixture>) } { uint32_t fieldId = f.getSchema().getIndexFieldId("i"); - PostingIterator itr(f._d.findFrozen("大灰" + PostingIterator itr(f._fic.findFrozen("大灰" "狼", fieldId), - featureStoreRef(f._d, fieldId), + featureStoreRef(f._fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_EQUAL(10u, itr.getDocId()); @@ -1434,7 +1437,7 @@ TEST_F("requireThatCjkIndexingIsWorking", DictionaryFixture<SingleFieldFixture>) void insertAndAssertTuple(const vespalib::string &word, uint32_t fieldId, uint32_t docId, - Dictionary &dict) + FieldIndexCollection &dict) { EntryRef wordRef = WrapInserter(dict, fieldId).rewind().word(word). add(docId).flush().getWordRef(); @@ -1445,7 +1448,7 @@ insertAndAssertTuple(const vespalib::string &word, uint32_t fieldId, uint32_t do TEST_F("require that insert tells which word ref that was inserted", Fixture) { - Dictionary d(f.getSchema()); + FieldIndexCollection d(f.getSchema()); insertAndAssertTuple("a", 1, 11, d); insertAndAssertTuple("b", 1, 11, d); insertAndAssertTuple("a", 2, 11, d); @@ -1457,14 +1460,14 @@ TEST_F("require that insert tells which word ref that was inserted", Fixture) struct RemoverFixture : public Fixture { - Dictionary _d; + FieldIndexCollection _fic; SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; RemoverFixture() : Fixture(), - _d(getSchema()), + _fic(getSchema()), _invertThreads(2), _pushThreads(2) { @@ -1472,24 +1475,24 @@ struct RemoverFixture : public Fixture void assertPostingLists(const vespalib::string &e1, const vespalib::string &e2, const vespalib::string &e3) { - EXPECT_TRUE(assertPostingList(e1, _d.find("a", 1))); - EXPECT_TRUE(assertPostingList(e2, _d.find("a", 2))); - EXPECT_TRUE(assertPostingList(e3, _d.find("b", 1))); + EXPECT_TRUE(assertPostingList(e1, _fic.find("a", 1))); + EXPECT_TRUE(assertPostingList(e2, _fic.find("a", 2))); + EXPECT_TRUE(assertPostingList(e3, _fic.find("b", 1))); } void remove(uint32_t docId) { DocumentInverter inv(getSchema(), _invertThreads, _pushThreads); - myremove(docId, inv, _d, _invertThreads); + myremove(docId, inv, _fic, _invertThreads); _pushThreads.sync(); - EXPECT_FALSE(_d.getFieldIndex(0u)->getDocumentRemover(). + EXPECT_FALSE(_fic.getFieldIndex(0u)->getDocumentRemover(). getStore().get(docId).valid()); } }; TEST_F("require that document remover can remove several documents", RemoverFixture) { - WrapInserter(f._d, 1).word("a").add(11).add(13).add(15). + WrapInserter(f._fic, 1).word("a").add(11).add(13).add(15). word("b").add(11).add(15).flush(); - WrapInserter(f._d, 2).word("a").add(11).add(13).flush(); + WrapInserter(f._fic, 2).word("a").add(11).add(13).flush(); f.assertPostingLists("[11,13,15]", "[11,13]", "[11,15]"); f.remove(13); @@ -1504,8 +1507,8 @@ TEST_F("require that document remover can remove several documents", RemoverFixt TEST_F("require that removal of non-existing document does not do anything", RemoverFixture) { - WrapInserter(f._d, 1).word("a").add(11).word("b").add(11).flush(); - WrapInserter(f._d, 2).word("a").add(11).flush(); + WrapInserter(f._fic, 1).word("a").add(11).word("b").add(11).flush(); + WrapInserter(f._fic, 2).word("a").add(11).flush(); f.assertPostingLists("[11]", "[11]", "[11]"); f.remove(13); f.assertPostingLists("[11]", "[11]", "[11]"); diff --git a/searchlib/src/tests/memoryindex/document_remover/.gitignore b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore index 2126f9147bd..2126f9147bd 100644 --- a/searchlib/src/tests/memoryindex/document_remover/.gitignore +++ b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore diff --git a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt new file mode 100644 index 00000000000..ef75337c6b6 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_index_remover_test_app TEST + SOURCES + field_index_remover_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_field_index_remover_test_app COMMAND searchlib_field_index_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp index af7a9422e49..fed6d963b70 100644 --- a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp @@ -2,9 +2,9 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/document_remover.h> -#include <vespa/searchlib/memoryindex/wordstore.h> -#include <vespa/searchlib/memoryindex/i_document_remove_listener.h> +#include <vespa/searchlib/memoryindex/field_index_remover.h> +#include <vespa/searchlib/memoryindex/i_field_index_remove_listener.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/test/insertion_operators.h> #include <algorithm> @@ -38,7 +38,7 @@ operator<<(std::ostream &os, const WordFieldPair &val) return os; } -struct MockRemoveListener : public IDocumentRemoveListener +struct MockRemoveListener : public IFieldIndexRemoveListener { WordFieldVector _words; uint32_t _expDocId; @@ -65,7 +65,7 @@ struct Fixture MockRemoveListener _listener; std::vector<std::unique_ptr<WordStore>> _wordStores; std::vector<std::map<vespalib::string, datastore::EntryRef>> _wordToRefMaps; - std::vector<std::unique_ptr<DocumentRemover>> _removers; + std::vector<std::unique_ptr<FieldIndexRemover>> _removers; Fixture() : _listener(), _wordStores(), @@ -75,7 +75,7 @@ struct Fixture uint32_t numFields = 4; for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { _wordStores.push_back(std::make_unique<WordStore>()); - _removers.push_back(std::make_unique<DocumentRemover> + _removers.push_back(std::make_unique<FieldIndexRemover> (*_wordStores.back())); } _wordToRefMaps.resize(numFields); diff --git a/searchlib/src/tests/memoryindex/field_inverter/.gitignore b/searchlib/src/tests/memoryindex/field_inverter/.gitignore new file mode 100644 index 00000000000..58579d09421 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..f39e05d6823 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_inverter_test_app TEST + SOURCES + field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_field_inverter_test_app COMMAND searchlib_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp index 1d066747ef8..f08e61b0da2 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -105,7 +105,7 @@ struct Fixture Schema _schema; DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore b/searchlib/src/tests/memoryindex/fieldinverter/.gitignore deleted file mode 100644 index 482663dd92e..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_fieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt deleted file mode 100644 index b6b223dceed..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_fieldinverter_test_app TEST - SOURCES - fieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_fieldinverter_test_app COMMAND searchlib_fieldinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/memory_index/.gitignore b/searchlib/src/tests/memoryindex/memory_index/.gitignore new file mode 100644 index 00000000000..6ec9ccf5015 --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +searchlib_memory_index_test_app diff --git a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt new file mode 100644 index 00000000000..d6b6691f05d --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_memory_index_test_app TEST + SOURCES + memory_index_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_memory_index_test_app COMMAND searchlib_memory_index_test_app) diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 9de6ac9f310..50f44074683 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -2,24 +2,24 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> -LOG_SETUP("memoryindex_test"); +LOG_SETUP("memory_index_test"); using document::Document; using document::FieldValue; diff --git a/searchlib/src/tests/memoryindex/memoryindex/.gitignore b/searchlib/src/tests/memoryindex/memoryindex/.gitignore deleted file mode 100644 index 174d0a494e2..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -memoryindex_test -sourceselectorwriter_test -searchlib_memoryindex_test_app diff --git a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt b/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt deleted file mode 100644 index 20f526e5c99..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_memoryindex_test_app TEST - SOURCES - memoryindex_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_memoryindex_test_app COMMAND searchlib_memoryindex_test_app) diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore new file mode 100644 index 00000000000..694dc947042 --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_url_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..28efc8a861e --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_url_field_inverter_test_app TEST + SOURCES + url_field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_url_field_inverter_test_app COMMAND searchlib_url_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp index daec09828f6..76fbf662b77 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* -*- mode: C++; coding: utf-8; -*- */ - +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/urlfieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/url_field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -183,7 +182,7 @@ struct Fixture DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; std::unique_ptr<UrlFieldInverter> _urlInverter; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; index::SchemaIndexFields _schemaIndexFields; static Schema diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore b/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore deleted file mode 100644 index b2636fe5e81..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_urlfieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt deleted file mode 100644 index 16fa8f5952e..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_urlfieldinverter_test_app TEST - SOURCES - urlfieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_urlfieldinverter_test_app COMMAND searchlib_urlfieldinverter_test_app) diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index 3619affb54e..b21b799e693 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -18,6 +18,8 @@ vespa_add_library(searchlib_diskindex OBJECT pagedict4file.cpp pagedict4randread.cpp wordnummapper.cpp + zc4_posting_writer.cpp + zc4_posting_writer_base.cpp zcbuf.cpp zcposocc.cpp zcposocciterators.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp new file mode 100644 index 00000000000..0eb59a383a5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp @@ -0,0 +1,270 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zc4_posting_writer.h" +#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/searchlib/index/postinglistcounts.h> + +using search::index::DocIdAndFeatures; +using search::index::PostingListCounts; +using search::index::PostingListParams; + +namespace search::diskindex +{ + +template <bool bigEndian> +Zc4PostingWriter<bigEndian>::Zc4PostingWriter(PostingListCounts &counts) + : Zc4PostingWriterBase(counts), + _encode_context(), + _encode_features(nullptr) +{ + _encode_context.setWriteContext(&_writeContext); + _writeContext.setEncodeContext(&_encode_context); +} + +template <bool bigEndian> +Zc4PostingWriter<bigEndian>::~Zc4PostingWriter() +{ +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::reset_chunk() +{ + _docIds.clear(); + if (_encode_features != nullptr) { + _encode_features->setupWrite(_featureWriteContext); + _featureOffset = 0; + } +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::flush_word_with_skip(bool hasMore) +{ + assert(_docIds.size() >= _minSkipDocs || !_counts._segments.empty()); + + if (_encode_features != nullptr) { + _encode_features->flush(); + } + EncodeContext &e = _encode_context; + + uint32_t numDocs = _docIds.size(); + + e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); + if (numDocs >= _minChunkDocs) { + e.writeBits((hasMore ? 1 : 0), 1); + } + + calc_skip_info(_encode_features != nullptr); + + uint32_t docIdsSize = _zcDocIds.size(); + uint32_t l1SkipSize = _l1Skip.size(); + uint32_t l2SkipSize = _l2Skip.size(); + uint32_t l3SkipSize = _l3Skip.size(); + uint32_t l4SkipSize = _l4Skip.size(); + + e.encodeExpGolomb(docIdsSize - 1, K_VALUE_ZCPOSTING_DOCIDSSIZE); + e.encodeExpGolomb(l1SkipSize, K_VALUE_ZCPOSTING_L1SKIPSIZE); + if (l1SkipSize != 0) { + e.encodeExpGolomb(l2SkipSize, K_VALUE_ZCPOSTING_L2SKIPSIZE); + if (l2SkipSize != 0) { + e.encodeExpGolomb(l3SkipSize, K_VALUE_ZCPOSTING_L3SKIPSIZE); + if (l3SkipSize != 0) { + e.encodeExpGolomb(l4SkipSize, K_VALUE_ZCPOSTING_L4SKIPSIZE); + } + } + } + if (_encode_features != nullptr) { + e.encodeExpGolomb(_featureOffset, K_VALUE_ZCPOSTING_FEATURESSIZE); + } + + // Encode last document id in chunk or word. + if (_dynamicK) { + uint32_t docIdK = e.calcDocIdK((_counts._segments.empty() && + !hasMore) ? + numDocs : 1, + _docIdLimit); + e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, + docIdK); + } else { + e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, + K_VALUE_ZCPOSTING_LASTDOCID); + } + + e.smallAlign(8); // Byte align + + uint8_t *docIds = _zcDocIds._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(docIds), + 0, + docIdsSize * 8); + if (l1SkipSize > 0) { + uint8_t *l1Skip = _l1Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l1Skip), + 0, + l1SkipSize * 8); + } + if (l2SkipSize > 0) { + uint8_t *l2Skip = _l2Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l2Skip), + 0, + l2SkipSize * 8); + } + if (l3SkipSize > 0) { + uint8_t *l3Skip = _l3Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l3Skip), + 0, + l3SkipSize * 8); + } + if (l4SkipSize > 0) { + uint8_t *l4Skip = _l4Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l4Skip), + 0, + l4SkipSize * 8); + } + + // Write features + e.writeBits(static_cast<const uint64_t *>(_featureWriteContext._comprBuf), + 0, + _featureOffset); + + _counts._numDocs += numDocs; + if (hasMore || !_counts._segments.empty()) { + uint64_t writePos = e.getWriteOffset(); + PostingListCounts::Segment seg; + seg._bitLength = writePos - (_writePos + _counts._bitLength); + seg._numDocs = numDocs; + seg._lastDoc = _docIds.back().first; + _counts._segments.push_back(seg); + _counts._bitLength += seg._bitLength; + } + // reset tables in preparation for next word or next chunk + clear_skip_info(); + reset_chunk(); +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::write_docid_and_features(const DocIdAndFeatures &features) +{ + if (__builtin_expect(_docIds.size() >= _minChunkDocs, false)) { + flush_word_with_skip(true); + } + if (_encode_features != nullptr) { + _encode_features->writeFeatures(features); + uint64_t writeOffset = _encode_features->getWriteOffset(); + uint64_t featureSize = writeOffset - _featureOffset; + assert(static_cast<uint32_t>(featureSize) == featureSize); + _docIds.push_back(std::make_pair(features._docId, + static_cast<uint32_t>(featureSize))); + _featureOffset = writeOffset; + } else { + _docIds.push_back(std::make_pair(features._docId, uint32_t(0))); + } +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::flush_word_no_skip() +{ + // Too few document ids for skip info. + assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); + + if (_encode_features != nullptr) { + _encode_features->flush(); + } + EncodeContext &e = _encode_context; + uint32_t numDocs = _docIds.size(); + + e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); + + uint32_t docIdK = _dynamicK ? e.calcDocIdK(numDocs, _docIdLimit) : K_VALUE_ZCPOSTING_DELTA_DOCID; + + uint32_t baseDocId = 1; + const uint64_t *features = + static_cast<const uint64_t *>(_featureWriteContext._comprBuf); + uint64_t featureOffset = 0; + + std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); + std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); + + for (; dit != dite; ++dit) { + uint32_t docId = dit->first; + uint32_t featureSize = dit->second; + e.encodeExpGolomb(docId - baseDocId, docIdK); + baseDocId = docId + 1; + if (featureSize != 0) { + e.writeBits(features + (featureOffset >> 6), + featureOffset & 63, + featureSize); + featureOffset += featureSize; + } + } + _counts._numDocs += numDocs; + reset_chunk(); +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::flush_word() +{ + if (__builtin_expect(_docIds.size() >= _minSkipDocs || + !_counts._segments.empty(), false)) { + // Use skip information if enough documents or chunking has happened + flush_word_with_skip(false); + _numWords++; + } else if (_docIds.size() > 0) { + flush_word_no_skip(); + _numWords++; + } + + EncodeContext &e = _encode_context; + uint64_t writePos = e.getWriteOffset(); + + _counts._bitLength = writePos - _writePos; + _writePos = writePos; +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::set_encode_features(EncodeContext *encode_features) +{ + _encode_features = encode_features; + if (_encode_features != nullptr) { + _encode_features->setWriteContext(&_featureWriteContext); + _encode_features->setupWrite(_featureWriteContext); + } + _featureWriteContext.setEncodeContext(_encode_features); + _featureOffset = 0; +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::on_open() +{ + _numWords = 0; + _writePos = _encode_context.getWriteOffset(); // Position after file header +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::on_close() +{ + // Write some pad bits to avoid decompression readahead going past + // memory mapped file during search and into SIGSEGV territory. + + // First pad to 64 bits alignment. + _encode_context.smallAlign(64); + _encode_context.writeComprBufferIfNeeded(); + + // Then write 128 more bits. This allows for 64-bit decoding + // with a readbits that always leaves a nonzero preRead + _encode_context.padBits(128); + _encode_context.alignDirectIO(); + _encode_context.flush(); + _encode_context.writeComprBuffer(); // Also flushes slack +} + +template class Zc4PostingWriter<false>; +template class Zc4PostingWriter<true>; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.h b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.h new file mode 100644 index 00000000000..8dc5e249d52 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.h @@ -0,0 +1,53 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "zc4_posting_writer_base.h" + +namespace search::index { class DocIdAndFeatures; } + +namespace search::diskindex +{ + +/* + * Class used to write posting lists of type "Zc.4" and "Zc.5" (dynamic k). + * + * Common words have docid deltas and skip info separate from + * features. + * + * Rare words do not have skip info, and docid deltas and features are + * interleaved. + */ +template <bool bigEndian> +class Zc4PostingWriter : public Zc4PostingWriterBase +{ + using EncodeContext = bitcompression::FeatureEncodeContext<bigEndian>; + + EncodeContext _encode_context; + // Buffer up features in memory + EncodeContext *_encode_features; +public: + Zc4PostingWriter(const Zc4PostingWriter &) = delete; + Zc4PostingWriter(Zc4PostingWriter &&) = delete; + Zc4PostingWriter &operator=(const Zc4PostingWriter &) = delete; + Zc4PostingWriter &operator=(Zc4PostingWriter &&) = delete; + Zc4PostingWriter(index::PostingListCounts &counts); + ~Zc4PostingWriter(); + + void reset_chunk(); + void flush_word_with_skip(bool hasMore); + void flush_word_no_skip(); + void flush_word(); + void write_docid_and_features(const index::DocIdAndFeatures &features); + void set_encode_features(EncodeContext *encode_features); + void on_open(); + void on_close(); + + EncodeContext &get_encode_features() { return *_encode_features; } + EncodeContext &get_encode_context() { return _encode_context; } +}; + +extern template class Zc4PostingWriter<false>; +extern template class Zc4PostingWriter<true>; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp new file mode 100644 index 00000000000..485610c2ebd --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp @@ -0,0 +1,222 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zc4_posting_writer_base.h" +#include <vespa/searchlib/index/postinglistcounts.h> + +using search::index::PostingListCounts; +using search::index::PostingListParams; + +namespace search::diskindex +{ + +Zc4PostingWriterBase::Zc4PostingWriterBase(PostingListCounts &counts) + : _minChunkDocs(1 << 30), + _minSkipDocs(64), + _docIdLimit(10000000), + _docIds(), + _featureOffset(0), + _writePos(0), + _dynamicK(false), + _zcDocIds(), + _l1Skip(), + _l2Skip(), + _l3Skip(), + _l4Skip(), + _numWords(0), + _counts(counts), + _writeContext(sizeof(uint64_t)), + _featureWriteContext(sizeof(uint64_t)) +{ + _featureWriteContext.allocComprBuf(64, 1); + // Ensure that some space is initially available in encoding buffers + _zcDocIds.maybeExpand(); + _l1Skip.maybeExpand(); + _l2Skip.maybeExpand(); + _l3Skip.maybeExpand(); + _l4Skip.maybeExpand(); +} + +Zc4PostingWriterBase::~Zc4PostingWriterBase() +{ +} + +#define L1SKIPSTRIDE 16 +#define L2SKIPSTRIDE 8 +#define L3SKIPSTRIDE 8 +#define L4SKIPSTRIDE 8 + +void +Zc4PostingWriterBase::calc_skip_info(bool encodeFeatures) +{ + uint32_t lastDocId = 0u; + uint32_t lastL1SkipDocId = 0u; + uint32_t lastL1SkipDocIdPos = 0; + uint32_t lastL1SkipFeaturePos = 0; + uint32_t lastL2SkipDocId = 0u; + uint32_t lastL2SkipDocIdPos = 0; + uint32_t lastL2SkipFeaturePos = 0; + uint32_t lastL2SkipL1SkipPos = 0; + uint32_t lastL3SkipDocId = 0u; + uint32_t lastL3SkipDocIdPos = 0; + uint32_t lastL3SkipFeaturePos = 0; + uint32_t lastL3SkipL1SkipPos = 0; + uint32_t lastL3SkipL2SkipPos = 0; + uint32_t lastL4SkipDocId = 0u; + uint32_t lastL4SkipDocIdPos = 0; + uint32_t lastL4SkipFeaturePos = 0; + uint32_t lastL4SkipL1SkipPos = 0; + uint32_t lastL4SkipL2SkipPos = 0; + uint32_t lastL4SkipL3SkipPos = 0; + unsigned int l1SkipCnt = 0; + unsigned int l2SkipCnt = 0; + unsigned int l3SkipCnt = 0; + unsigned int l4SkipCnt = 0; + uint64_t featurePos = 0; + + std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); + std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); + + if (!_counts._segments.empty()) { + lastDocId = _counts._segments.back()._lastDoc; + lastL1SkipDocId = lastDocId; + lastL2SkipDocId = lastDocId; + lastL3SkipDocId = lastDocId; + lastL4SkipDocId = lastDocId; + } + + for (; dit != dite; ++dit) { + if (l1SkipCnt >= L1SKIPSTRIDE) { + // L1 docid delta + uint32_t docIdDelta = lastDocId - lastL1SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l1Skip.encode(docIdDelta - 1); + lastL1SkipDocId = lastDocId; + // L1 docid pos + uint64_t docIdPos = _zcDocIds.size(); + _l1Skip.encode(docIdPos - lastL1SkipDocIdPos - 1); + lastL1SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L1 features pos + _l1Skip.encode(featurePos - lastL1SkipFeaturePos - 1); + lastL1SkipFeaturePos = featurePos; + } + l1SkipCnt = 0; + ++l2SkipCnt; + if (l2SkipCnt >= L2SKIPSTRIDE) { + // L2 docid delta + docIdDelta = lastDocId - lastL2SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l2Skip.encode(docIdDelta - 1); + lastL2SkipDocId = lastDocId; + // L2 docid pos + docIdPos = _zcDocIds.size(); + _l2Skip.encode(docIdPos - lastL2SkipDocIdPos - 1); + lastL2SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L2 features pos + _l2Skip.encode(featurePos - lastL2SkipFeaturePos - 1); + lastL2SkipFeaturePos = featurePos; + } + // L2 L1Skip pos + uint64_t l1SkipPos = _l1Skip.size(); + _l2Skip.encode(l1SkipPos - lastL2SkipL1SkipPos - 1); + lastL2SkipL1SkipPos = l1SkipPos; + l2SkipCnt = 0; + ++l3SkipCnt; + if (l3SkipCnt >= L3SKIPSTRIDE) { + // L3 docid delta + docIdDelta = lastDocId - lastL3SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l3Skip.encode(docIdDelta - 1); + lastL3SkipDocId = lastDocId; + // L3 docid pos + docIdPos = _zcDocIds.size(); + _l3Skip.encode(docIdPos - lastL3SkipDocIdPos - 1); + lastL3SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L3 features pos + _l3Skip.encode(featurePos - lastL3SkipFeaturePos - 1); + lastL3SkipFeaturePos = featurePos; + } + // L3 L1Skip pos + l1SkipPos = _l1Skip.size(); + _l3Skip.encode(l1SkipPos - lastL3SkipL1SkipPos - 1); + lastL3SkipL1SkipPos = l1SkipPos; + // L3 L2Skip pos + uint64_t l2SkipPos = _l2Skip.size(); + _l3Skip.encode(l2SkipPos - lastL3SkipL2SkipPos - 1); + lastL3SkipL2SkipPos = l2SkipPos; + l3SkipCnt = 0; + ++l4SkipCnt; + if (l4SkipCnt >= L4SKIPSTRIDE) { + // L4 docid delta + docIdDelta = lastDocId - lastL4SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l4Skip.encode(docIdDelta - 1); + lastL4SkipDocId = lastDocId; + // L4 docid pos + docIdPos = _zcDocIds.size(); + _l4Skip.encode(docIdPos - lastL4SkipDocIdPos - 1); + lastL4SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L4 features pos + _l4Skip.encode(featurePos - lastL4SkipFeaturePos - 1); + lastL4SkipFeaturePos = featurePos; + } + // L4 L1Skip pos + l1SkipPos = _l1Skip.size(); + _l4Skip.encode(l1SkipPos - lastL4SkipL1SkipPos - 1); + lastL4SkipL1SkipPos = l1SkipPos; + // L4 L2Skip pos + l2SkipPos = _l2Skip.size(); + _l4Skip.encode(l2SkipPos - lastL4SkipL2SkipPos - 1); + lastL4SkipL2SkipPos = l2SkipPos; + // L4 L3Skip pos + uint64_t l3SkipPos = _l3Skip.size(); + _l4Skip.encode(l3SkipPos - lastL4SkipL3SkipPos - 1); + lastL4SkipL3SkipPos = l3SkipPos; + l4SkipCnt = 0; + } + } + } + } + uint32_t docId = dit->first; + featurePos += dit->second; + _zcDocIds.encode(docId - lastDocId - 1); + lastDocId = docId; + ++l1SkipCnt; + } + // Extra partial entries for skip tables to simplify iterator during search + if (_l1Skip.size() > 0) { + _l1Skip.encode(lastDocId - lastL1SkipDocId - 1); + } + if (_l2Skip.size() > 0) { + _l2Skip.encode(lastDocId - lastL2SkipDocId - 1); + } + if (_l3Skip.size() > 0) { + _l3Skip.encode(lastDocId - lastL3SkipDocId - 1); + } + if (_l4Skip.size() > 0) { + _l4Skip.encode(lastDocId - lastL4SkipDocId - 1); + } +} + +void +Zc4PostingWriterBase::clear_skip_info() +{ + _zcDocIds.clear(); + _l1Skip.clear(); + _l2Skip.clear(); + _l3Skip.clear(); + _l4Skip.clear(); +} + +void +Zc4PostingWriterBase::set_posting_list_params(const PostingListParams ¶ms) +{ + params.get("docIdLimit", _docIdLimit); + params.get("minChunkDocs", _minChunkDocs); + params.get("minSkipDocs", _minSkipDocs); +} + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h new file mode 100644 index 00000000000..ba781c11564 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h @@ -0,0 +1,66 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "zcbuf.h" +#include <vespa/searchlib/bitcompression/compression.h> +#include <vector> + +namespace search::index { +class PostingListCounts; +class PostingListParams; +} + +namespace search::diskindex +{ + +/* + * Base class for writing posting lists that might have basic skip info. + */ +class Zc4PostingWriterBase +{ +protected: + uint32_t _minChunkDocs; // # of documents needed for chunking + uint32_t _minSkipDocs; // # of documents needed for skipping + uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) + + // Unpacked document ids for word and feature sizes + using DocIdAndFeatureSize = std::pair<uint32_t, uint32_t>; + std::vector<DocIdAndFeatureSize> _docIds; + + uint64_t _featureOffset; // Bit offset of next feature + uint64_t _writePos; // Bit position for start of current word + bool _dynamicK; // Caclulate EG compression parameters ? + ZcBuf _zcDocIds; // Document id deltas + ZcBuf _l1Skip; // L1 skip info + ZcBuf _l2Skip; // L2 skip info + ZcBuf _l3Skip; // L3 skip info + ZcBuf _l4Skip; // L4 skip info + + uint64_t _numWords; // Number of words in file + index::PostingListCounts &_counts; + search::ComprFileWriteContext _writeContext; + search::ComprFileWriteContext _featureWriteContext; + + Zc4PostingWriterBase(const Zc4PostingWriterBase &) = delete; + Zc4PostingWriterBase(Zc4PostingWriterBase &&) = delete; + Zc4PostingWriterBase &operator=(const Zc4PostingWriterBase &) = delete; + Zc4PostingWriterBase &operator=(Zc4PostingWriterBase &&) = delete; + Zc4PostingWriterBase(index::PostingListCounts &counts); + ~Zc4PostingWriterBase(); + void calc_skip_info(bool encodeFeatures); + void clear_skip_info(); + +public: + ComprFileWriteContext &get_write_context() { return _writeContext; } + ComprFileWriteContext &get_feature_write_context() { return _featureWriteContext; } + uint32_t get_min_chunk_docs() const { return _minChunkDocs; } + uint32_t get_min_skip_docs() const { return _minSkipDocs; } + uint32_t get_docid_limit() const { return _docIdLimit; } + uint64_t get_num_words() const { return _numWords; } + bool get_dynamic_k() const { return _dynamicK; } + void set_dynamic_k(bool dynamicK) { _dynamicK = dynamicK; } + void set_posting_list_params(const index::PostingListParams ¶ms); +}; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp index df06432816f..10c08af92cb 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp @@ -63,9 +63,7 @@ Zc4PosOccSeqWrite::Zc4PosOccSeqWrite(const Schema &schema, _fieldsParams(), _realEncodeFeatures(&_fieldsParams) { - _encodeFeatures = &_realEncodeFeatures; - _encodeFeatures->setWriteContext(&_featureWriteContext); - _featureWriteContext.setEncodeContext(_encodeFeatures); + _writer.set_encode_features(&_realEncodeFeatures); _fieldsParams.setSchemaParams(schema, indexId); } @@ -118,9 +116,7 @@ ZcPosOccSeqWrite::ZcPosOccSeqWrite(const Schema &schema, _fieldsParams(), _realEncodeFeatures(&_fieldsParams) { - _encodeFeatures = &_realEncodeFeatures; - _encodeFeatures->setWriteContext(&_featureWriteContext); - _featureWriteContext.setEncodeContext(_encodeFeatures); + _writer.set_encode_features(&_realEncodeFeatures); _fieldsParams.setSchemaParams(schema, indexId); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index d51a592bf2b..e850f169adc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -607,36 +607,16 @@ Zc4PostingSeqRead::setPostingOffset(uint64_t offset, Zc4PostingSeqWrite:: Zc4PostingSeqWrite(PostingListCountFileSeqWrite *countFile) : PostingListFileSeqWrite(), - _encodeContext(), - _writeContext(_encodeContext), + _writer(_counts), _file(), - _minChunkDocs(1 << 30), - _minSkipDocs(64), - _docIdLimit(10000000), - _docIds(), - _encodeFeatures(nullptr), - _featureOffset(0), - _featureWriteContext(sizeof(uint64_t)), - _writePos(0), - _dynamicK(false), - _zcDocIds(), - _l1Skip(), - _l2Skip(), - _l3Skip(), - _l4Skip(), - _numWords(0), _fileBitSize(0), _countFile(countFile) { - _encodeContext.setWriteContext(&_writeContext); - if (_countFile != nullptr) { PostingListParams params; _countFile->getParams(params); - params.get("docIdLimit", _docIdLimit); - params.get("minChunkDocs", _minChunkDocs); + _writer.set_posting_list_params(params); } - _featureWriteContext.allocComprBuf(64, 1); } @@ -646,110 +626,27 @@ Zc4PostingSeqWrite::~Zc4PostingSeqWrite() void -Zc4PostingSeqWrite:: -writeDocIdAndFeatures(const DocIdAndFeatures &features) +Zc4PostingSeqWrite::writeDocIdAndFeatures(const DocIdAndFeatures &features) { - if (__builtin_expect(_docIds.size() >= _minChunkDocs, false)) - flushChunk(); - _encodeFeatures->writeFeatures(features); - uint64_t writeOffset = _encodeFeatures->getWriteOffset(); - uint64_t featureSize = writeOffset - _featureOffset; - assert(static_cast<uint32_t>(featureSize) == featureSize); - _docIds.push_back(std::make_pair(features._docId, - static_cast<uint32_t>(featureSize))); - _featureOffset = writeOffset; + _writer.write_docid_and_features(features); } void Zc4PostingSeqWrite::flushWord() { - if (__builtin_expect(_docIds.size() >= _minSkipDocs || - !_counts._segments.empty(), false)) { - // Use skip information if enough documents of chunking has happened - flushWordWithSkip(false); - _numWords++; - } else if (_docIds.size() > 0) { - flushWordNoSkip(); - _numWords++; - } - - EncodeContext &e = _encodeContext; - uint64_t writePos = e.getWriteOffset(); - - _counts._bitLength = writePos - _writePos; - _writePos = writePos; -} - - -uint32_t -Zc4PostingSeqWrite::readHeader(const vespalib::string &name) -{ - EncodeContext &f = *_encodeFeatures; - - FeatureDecodeContextBE d; - ComprFileReadContext drc(d); - FastOS_File file; - const vespalib::string &myId = _dynamicK ? myId5 : myId4; - - d.setReadContext(&drc); - bool res = file.OpenReadOnly(name.c_str()); - if (!res) { - LOG(error, "Could not open %s for reading file header: %s", - name.c_str(), getLastErrorString().c_str()); - LOG_ABORT("should not be reached"); - } - - drc.setFile(&file); - drc.setFileSize(file.GetSize()); - drc.allocComprBuf(512, 32768u); - d.emptyBuffer(0); - drc.readComprBuffer(); - - vespalib::FileHeader header; - d.readHeader(header, file.getSize()); - uint32_t headerLen = header.getSize(); - assert(header.hasTag("frozen")); - assert(header.hasTag("fileBitSize")); - assert(header.hasTag("format.0")); - assert(header.hasTag("format.1")); - assert(!header.hasTag("format.2")); - assert(header.hasTag("numWords")); - assert(header.hasTag("minChunkDocs")); - assert(header.hasTag("docIdLimit")); - assert(header.hasTag("minSkipDocs")); - assert(header.hasTag("endian")); - bool headerCompleted = header.getTag("frozen").asInteger() != 0; - uint64_t headerFileBitSize = header.getTag("fileBitSize").asInteger(); - headerLen += (-headerLen & 7); - assert(!headerCompleted || headerFileBitSize >= headerLen * 8); - (void) headerCompleted; - (void) headerFileBitSize; - assert(header.getTag("format.0").asString() == myId); - (void) myId; - assert(header.getTag("format.1").asString() == f.getIdentifier()); - _minChunkDocs = header.getTag("minChunkDocs").asInteger(); - _docIdLimit = header.getTag("docIdLimit").asInteger(); - _minSkipDocs = header.getTag("minSkipDocs").asInteger(); - assert(header.getTag("endian").asString() == "big"); - // Read feature decoding specific subheader using helper decode context - f.readHeader(header, "features."); - // Align on 64-bit unit - d.smallAlign(64); - assert(d.getReadOffset() == headerLen * 8); - file.Close(); - return headerLen; + _writer.flush_word(); } void Zc4PostingSeqWrite::makeHeader(const FileHeaderContext &fileHeaderContext) { - EncodeContext &f = *_encodeFeatures; - EncodeContext &e = _encodeContext; - ComprFileWriteContext &wce = _writeContext; + EncodeContext &f = _writer.get_encode_features(); + EncodeContext &e = _writer.get_encode_context(); + ComprFileWriteContext &wce = _writer.get_write_context(); - const vespalib::string &myId = _dynamicK ? myId5 : myId4; + const vespalib::string &myId = _writer.get_dynamic_k() ? myId5 : myId4; vespalib::FileHeader header; typedef vespalib::GenericHeader::Tag Tag; @@ -759,9 +656,9 @@ Zc4PostingSeqWrite::makeHeader(const FileHeaderContext &fileHeaderContext) header.putTag(Tag("format.0", myId)); header.putTag(Tag("format.1", f.getIdentifier())); header.putTag(Tag("numWords", 0)); - header.putTag(Tag("minChunkDocs", _minChunkDocs)); - header.putTag(Tag("docIdLimit", _docIdLimit)); - header.putTag(Tag("minSkipDocs", _minSkipDocs)); + header.putTag(Tag("minChunkDocs", _writer.get_min_chunk_docs())); + header.putTag(Tag("docIdLimit", _writer.get_docid_limit())); + header.putTag(Tag("minSkipDocs", _writer.get_min_skip_docs())); header.putTag(Tag("endian", "big")); header.putTag(Tag("desc", "Posting list file")); @@ -788,7 +685,7 @@ Zc4PostingSeqWrite::updateHeader() typedef vespalib::GenericHeader::Tag Tag; h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", _fileBitSize)); - h.putTag(Tag("numWords", _numWords)); + h.putTag(Tag("numWords", _writer.get_num_words())); h.rewriteFile(f); f.Sync(); f.Close(); @@ -813,40 +710,21 @@ Zc4PostingSeqWrite::open(const vespalib::string &name, // XXX may need to do something more here, I don't know what... return false; } - uint64_t fileSize = _file.GetSize(); - uint64_t bufferStartFilePos = _writeContext.getBufferStartFilePos(); - assert(fileSize >= bufferStartFilePos); - (void) fileSize; - _file.SetSize(bufferStartFilePos); - assert(bufferStartFilePos == static_cast<uint64_t>(_file.GetPosition())); - _writeContext.setFile(&_file); - search::ComprBuffer &cb = _writeContext; - EncodeContext &e = _encodeContext; - _writeContext.allocComprBuf(65536u, 32768u); - if (bufferStartFilePos == 0) { - e.setupWrite(cb); - // Reset accumulated stats - _fileBitSize = 0; - _numWords = 0; - // Start write initial header - makeHeader(fileHeaderContext); - _encodeFeatures->setupWrite(_featureWriteContext); - // end write initial header - _writePos = e.getWriteOffset(); - } else { - assert(bufferStartFilePos >= 8u); - uint32_t headerSize = readHeader(name); // Read existing header - assert(bufferStartFilePos >= headerSize); - (void) headerSize; - e.afterWrite(_writeContext, 0, bufferStartFilePos); - } - - // Ensure that some space is initially available in encoding buffers - _zcDocIds.maybeExpand(); - _l1Skip.maybeExpand(); - _l2Skip.maybeExpand(); - _l3Skip.maybeExpand(); - _l4Skip.maybeExpand(); + auto &writeContext = _writer.get_write_context(); + uint64_t bufferStartFilePos = writeContext.getBufferStartFilePos(); + assert(bufferStartFilePos == 0); + _file.SetSize(0); + writeContext.setFile(&_file); + search::ComprBuffer &cb = writeContext; + EncodeContext &e = _writer.get_encode_context(); + writeContext.allocComprBuf(65536u, 32768u); + e.setupWrite(cb); + // Reset accumulated stats + _fileBitSize = 0; + // Start write initial header + makeHeader(fileHeaderContext); + // end write initial header + _writer.on_open(); return true; // Assume success } @@ -854,42 +732,24 @@ Zc4PostingSeqWrite::open(const vespalib::string &name, bool Zc4PostingSeqWrite::close() { - EncodeContext &e = _encodeContext; - - _fileBitSize = e.getWriteOffset(); - // Write some pad bits to avoid decompression readahead going past - // memory mapped file during search and into SIGSEGV territory. - - // First pad to 64 bits alignment. - e.smallAlign(64); - e.writeComprBufferIfNeeded(); - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - e.padBits(128); - e.alignDirectIO(); - e.flush(); - e.writeComprBuffer(); // Also flushes slack - - _writeContext.dropComprBuf(); + _fileBitSize = _writer.get_encode_context().getWriteOffset(); + _writer.on_close(); // flush and pad + auto &writeContext = _writer.get_write_context(); + writeContext.dropComprBuf(); _file.Sync(); _file.Close(); - _writeContext.setFile(nullptr); + writeContext.setFile(nullptr); updateHeader(); return true; } - - void Zc4PostingSeqWrite:: setParams(const PostingListParams ¶ms) { if (_countFile != nullptr) _countFile->setParams(params); - params.get("docIdLimit", _docIdLimit); - params.get("minChunkDocs", _minChunkDocs); - params.get("minSkipDocs", _minSkipDocs); + _writer.set_posting_list_params(params); } @@ -905,14 +765,14 @@ getParams(PostingListParams ¶ms) uint32_t countMinChunkDocs = 0; countParams.get("docIdLimit", countDocIdLimit); countParams.get("minChunkDocs", countMinChunkDocs); - assert(_docIdLimit == countDocIdLimit); - assert(_minChunkDocs == countMinChunkDocs); + assert(_writer.get_docid_limit() == countDocIdLimit); + assert(_writer.get_min_chunk_docs() == countMinChunkDocs); } else { params.clear(); - params.set("docIdLimit", _docIdLimit); - params.set("minChunkDocs", _minChunkDocs); + params.set("docIdLimit", _writer.get_docid_limit()); + params.set("minChunkDocs", _writer.get_min_chunk_docs()); } - params.set("minSkipDocs", _minSkipDocs); + params.set("minSkipDocs", _writer.get_min_skip_docs()); } @@ -920,7 +780,7 @@ void Zc4PostingSeqWrite:: setFeatureParams(const PostingListParams ¶ms) { - _encodeFeatures->setParams(params); + _writer.get_encode_features().setParams(params); } @@ -928,314 +788,7 @@ void Zc4PostingSeqWrite:: getFeatureParams(PostingListParams ¶ms) { - _encodeFeatures->getParams(params); -} - - -void -Zc4PostingSeqWrite::flushChunk() -{ - /* TODO: Flush chunk and prepare for new (possible short) chunk */ - flushWordWithSkip(true); -} - -#define L1SKIPSTRIDE 16 -#define L2SKIPSTRIDE 8 -#define L3SKIPSTRIDE 8 -#define L4SKIPSTRIDE 8 - - -void -Zc4PostingSeqWrite::calcSkipInfo() -{ - uint32_t lastDocId = 0u; - uint32_t lastL1SkipDocId = 0u; - uint32_t lastL1SkipDocIdPos = 0; - uint32_t lastL1SkipFeaturePos = 0; - uint32_t lastL2SkipDocId = 0u; - uint32_t lastL2SkipDocIdPos = 0; - uint32_t lastL2SkipFeaturePos = 0; - uint32_t lastL2SkipL1SkipPos = 0; - uint32_t lastL3SkipDocId = 0u; - uint32_t lastL3SkipDocIdPos = 0; - uint32_t lastL3SkipFeaturePos = 0; - uint32_t lastL3SkipL1SkipPos = 0; - uint32_t lastL3SkipL2SkipPos = 0; - uint32_t lastL4SkipDocId = 0u; - uint32_t lastL4SkipDocIdPos = 0; - uint32_t lastL4SkipFeaturePos = 0; - uint32_t lastL4SkipL1SkipPos = 0; - uint32_t lastL4SkipL2SkipPos = 0; - uint32_t lastL4SkipL3SkipPos = 0; - unsigned int l1SkipCnt = 0; - unsigned int l2SkipCnt = 0; - unsigned int l3SkipCnt = 0; - unsigned int l4SkipCnt = 0; - uint64_t featurePos = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - - if (!_counts._segments.empty()) { - lastDocId = _counts._segments.back()._lastDoc; - lastL1SkipDocId = lastDocId; - lastL2SkipDocId = lastDocId; - lastL3SkipDocId = lastDocId; - lastL4SkipDocId = lastDocId; - } - - for (; dit != dite; ++dit) { - if (l1SkipCnt >= L1SKIPSTRIDE) { - // L1 docid delta - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l1Skip.encode(docIdDelta - 1); - lastL1SkipDocId = lastDocId; - // L1 docid pos - uint64_t docIdPos = _zcDocIds.size(); - _l1Skip.encode(docIdPos - lastL1SkipDocIdPos - 1); - lastL1SkipDocIdPos = docIdPos; - // L1 features pos - _l1Skip.encode(featurePos - lastL1SkipFeaturePos - 1); - lastL1SkipFeaturePos = featurePos; - l1SkipCnt = 0; - ++l2SkipCnt; - if (l2SkipCnt >= L2SKIPSTRIDE) { - // L2 docid delta - docIdDelta = lastDocId - lastL2SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l2Skip.encode(docIdDelta - 1); - lastL2SkipDocId = lastDocId; - // L2 docid pos - docIdPos = _zcDocIds.size(); - _l2Skip.encode(docIdPos - lastL2SkipDocIdPos - 1); - lastL2SkipDocIdPos = docIdPos; - // L2 features pos - _l2Skip.encode(featurePos - lastL2SkipFeaturePos - 1); - lastL2SkipFeaturePos = featurePos; - // L2 L1Skip pos - uint64_t l1SkipPos = _l1Skip.size(); - _l2Skip.encode(l1SkipPos - lastL2SkipL1SkipPos - 1); - lastL2SkipL1SkipPos = l1SkipPos; - l2SkipCnt = 0; - ++l3SkipCnt; - if (l3SkipCnt >= L3SKIPSTRIDE) { - // L3 docid delta - docIdDelta = lastDocId - lastL3SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l3Skip.encode(docIdDelta - 1); - lastL3SkipDocId = lastDocId; - // L3 docid pos - docIdPos = _zcDocIds.size(); - _l3Skip.encode(docIdPos - lastL3SkipDocIdPos - 1); - lastL3SkipDocIdPos = docIdPos; - // L3 features pos - _l3Skip.encode(featurePos - lastL3SkipFeaturePos - 1); - lastL3SkipFeaturePos = featurePos; - // L3 L1Skip pos - l1SkipPos = _l1Skip.size(); - _l3Skip.encode(l1SkipPos - lastL3SkipL1SkipPos - 1); - lastL3SkipL1SkipPos = l1SkipPos; - // L3 L2Skip pos - uint64_t l2SkipPos = _l2Skip.size(); - _l3Skip.encode(l2SkipPos - lastL3SkipL2SkipPos - 1); - lastL3SkipL2SkipPos = l2SkipPos; - l3SkipCnt = 0; - ++l4SkipCnt; - if (l4SkipCnt >= L4SKIPSTRIDE) { - // L4 docid delta - docIdDelta = lastDocId - lastL4SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l4Skip.encode(docIdDelta - 1); - lastL4SkipDocId = lastDocId; - // L4 docid pos - docIdPos = _zcDocIds.size(); - _l4Skip.encode(docIdPos - lastL4SkipDocIdPos - 1); - lastL4SkipDocIdPos = docIdPos; - // L4 features pos - _l4Skip.encode(featurePos - lastL4SkipFeaturePos - 1); - lastL4SkipFeaturePos = featurePos; - // L4 L1Skip pos - l1SkipPos = _l1Skip.size(); - _l4Skip.encode(l1SkipPos - lastL4SkipL1SkipPos - 1); - lastL4SkipL1SkipPos = l1SkipPos; - // L4 L2Skip pos - l2SkipPos = _l2Skip.size(); - _l4Skip.encode(l2SkipPos - lastL4SkipL2SkipPos - 1); - lastL4SkipL2SkipPos = l2SkipPos; - // L4 L3Skip pos - uint64_t l3SkipPos = _l3Skip.size(); - _l4Skip.encode(l3SkipPos - lastL4SkipL3SkipPos - 1); - lastL4SkipL3SkipPos = l3SkipPos; - l4SkipCnt = 0; - } - } - } - } - uint32_t docId = dit->first; - featurePos += dit->second; - _zcDocIds.encode(docId - lastDocId - 1); - lastDocId = docId; - ++l1SkipCnt; - } - // Extra partial entries for skip tables to simplify iterator during search - if (_l1Skip.size() > 0) - _l1Skip.encode(lastDocId - lastL1SkipDocId - 1); - if (_l2Skip.size() > 0) - _l2Skip.encode(lastDocId - lastL2SkipDocId - 1); - if (_l3Skip.size() > 0) - _l3Skip.encode(lastDocId - lastL3SkipDocId - 1); - if (_l4Skip.size() > 0) - _l4Skip.encode(lastDocId - lastL4SkipDocId - 1); -} - - -void -Zc4PostingSeqWrite::flushWordWithSkip(bool hasMore) -{ - assert(_docIds.size() >= _minSkipDocs || !_counts._segments.empty()); - - _encodeFeatures->flush(); - EncodeContext &e = _encodeContext; - - uint32_t numDocs = _docIds.size(); - - e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - if (numDocs >= _minChunkDocs) - e.writeBits((hasMore ? 1 : 0), 1); - - // TODO: Calculate docids size, possible also k parameter */ - calcSkipInfo(); - - uint32_t docIdsSize = _zcDocIds.size(); - uint32_t l1SkipSize = _l1Skip.size(); - uint32_t l2SkipSize = _l2Skip.size(); - uint32_t l3SkipSize = _l3Skip.size(); - uint32_t l4SkipSize = _l4Skip.size(); - - e.encodeExpGolomb(docIdsSize - 1, K_VALUE_ZCPOSTING_DOCIDSSIZE); - e.encodeExpGolomb(l1SkipSize, K_VALUE_ZCPOSTING_L1SKIPSIZE); - if (l1SkipSize != 0) { - e.encodeExpGolomb(l2SkipSize, K_VALUE_ZCPOSTING_L2SKIPSIZE); - if (l2SkipSize != 0) { - e.encodeExpGolomb(l3SkipSize, K_VALUE_ZCPOSTING_L3SKIPSIZE); - if (l3SkipSize != 0) { - e.encodeExpGolomb(l4SkipSize, K_VALUE_ZCPOSTING_L4SKIPSIZE); - } - } - } - e.encodeExpGolomb(_featureOffset, K_VALUE_ZCPOSTING_FEATURESSIZE); - - // Encode last document id in chunk or word. - if (_dynamicK) { - uint32_t docIdK = e.calcDocIdK((_counts._segments.empty() && - !hasMore) ? - numDocs : 1, - _docIdLimit); - e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, - docIdK); - } else { - e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, - K_VALUE_ZCPOSTING_LASTDOCID); - } - - e.smallAlign(8); // Byte align - - uint8_t *docIds = _zcDocIds._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(docIds), - 0, - docIdsSize * 8); - if (l1SkipSize > 0) { - uint8_t *l1Skip = _l1Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l1Skip), - 0, - l1SkipSize * 8); - } - if (l2SkipSize > 0) { - uint8_t *l2Skip = _l2Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l2Skip), - 0, - l2SkipSize * 8); - } - if (l3SkipSize > 0) { - uint8_t *l3Skip = _l3Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l3Skip), - 0, - l3SkipSize * 8); - } - if (l4SkipSize > 0) { - uint8_t *l4Skip = _l4Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l4Skip), - 0, - l4SkipSize * 8); - } - - // Write features - e.writeBits(static_cast<const uint64_t *>(_featureWriteContext._comprBuf), - 0, - _featureOffset); - - _counts._numDocs += numDocs; - if (hasMore || !_counts._segments.empty()) { - uint64_t writePos = e.getWriteOffset(); - PostingListCounts::Segment seg; - seg._bitLength = writePos - (_writePos + _counts._bitLength); - seg._numDocs = numDocs; - seg._lastDoc = _docIds.back().first; - _counts._segments.push_back(seg); - _counts._bitLength += seg._bitLength; - } - // reset tables in preparation for next word or next chunk - _zcDocIds.clear(); - _l1Skip.clear(); - _l2Skip.clear(); - _l3Skip.clear(); - _l4Skip.clear(); - resetWord(); -} - - -void -Zc4PostingSeqWrite::flushWordNoSkip() -{ - // Too few document ids for skip info. - assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); - - _encodeFeatures->flush(); - EncodeContext &e = _encodeContext; - uint32_t numDocs = _docIds.size(); - - e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - - uint32_t baseDocId = 1; - const uint64_t *features = - static_cast<const uint64_t *>(_featureWriteContext._comprBuf); - uint64_t featureOffset = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - - for (; dit != dite; ++dit) { - uint32_t docId = dit->first; - uint32_t featureSize = dit->second; - e.encodeExpGolomb(docId - baseDocId, K_VALUE_ZCPOSTING_DELTA_DOCID); - baseDocId = docId + 1; - e.writeBits(features + (featureOffset >> 6), - featureOffset & 63, - featureSize); - featureOffset += featureSize; - } - _counts._numDocs += numDocs; - resetWord(); -} - - -void -Zc4PostingSeqWrite::resetWord() -{ - _docIds.clear(); - _encodeFeatures->setupWrite(_featureWriteContext); - _featureOffset = 0; + _writer.get_encode_features().getParams(params); } @@ -1300,44 +853,7 @@ ZcPostingSeqRead::getIdentifier() ZcPostingSeqWrite::ZcPostingSeqWrite(PostingListCountFileSeqWrite *countFile) : Zc4PostingSeqWrite(countFile) { - _dynamicK = true; -} - - -void -ZcPostingSeqWrite::flushWordNoSkip() -{ - // Too few document ids for skip info. - assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); - - _encodeFeatures->flush(); - EncodeContext &e = _encodeContext; - uint32_t numDocs = _docIds.size(); - - e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - - uint32_t docIdK = e.calcDocIdK(numDocs, _docIdLimit); - - uint32_t baseDocId = 1; - const uint64_t *features = - static_cast<const uint64_t *>(_featureWriteContext._comprBuf); - uint64_t featureOffset = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - - for (; dit != dite; ++dit) { - uint32_t docId = dit->first; - uint32_t featureSize = dit->second; - e.encodeExpGolomb(docId - baseDocId, docIdK); - baseDocId = docId + 1; - e.writeBits(features + (featureOffset >> 6), - featureOffset & 63, - featureSize); - featureOffset += featureSize; - } - _counts._numDocs += numDocs; - resetWord(); + _writer.set_dynamic_k(true); } } // namespace search::diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 8c69a051e83..96cc306cea8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -2,9 +2,8 @@ #pragma once -#include "zcbuf.h" +#include "zc4_posting_writer.h" #include <vespa/searchlib/index/postinglistfile.h> -#include <vespa/searchlib/bitcompression/compression.h> #include <vespa/fastos/file.h> namespace search::index { @@ -131,29 +130,8 @@ class Zc4PostingSeqWrite : public index::PostingListFileSeqWrite protected: typedef bitcompression::FeatureEncodeContextBE EncodeContext; - EncodeContext _encodeContext; - search::ComprFileWriteContext _writeContext; - FastOS_File _file; - uint32_t _minChunkDocs; // # of documents needed for chunking - uint32_t _minSkipDocs; // # of documents needed for skipping - uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) - // Unpacked document ids for word and feature sizes - typedef std::pair<uint32_t, uint32_t> DocIdAndFeatureSize; - std::vector<DocIdAndFeatureSize> _docIds; - - // Buffer up features in memory - EncodeContext *_encodeFeatures; - uint64_t _featureOffset; // Bit offset of next feature - search::ComprFileWriteContext _featureWriteContext; - uint64_t _writePos; // Bit position for start of current word - bool _dynamicK; // Caclulate EG compression parameters ? - ZcBuf _zcDocIds; // Document id deltas - ZcBuf _l1Skip; // L1 skip info - ZcBuf _l2Skip; // L2 skip info - ZcBuf _l3Skip; // L3 skip info - ZcBuf _l4Skip; // L4 skip info - - uint64_t _numWords; // Number of words in file + Zc4PostingWriter<true> _writer; + FastOS_File _file; uint64_t _fileBitSize; index::PostingListCountFileSeqWrite *const _countFile; public: @@ -177,37 +155,10 @@ public: void getFeatureParams(PostingListParams ¶ms) override; /** - * Flush chunk to file. - */ - void flushChunk(); - void calcSkipInfo(); - - /** - * Flush word with skip info to disk - */ - void flushWordWithSkip(bool hasMore); - - - /** - * Flush word without skip info to disk. - */ - virtual void flushWordNoSkip(); - - /** - * Prepare for next word or next chunk. - */ - void resetWord(); - - /** * Make header using feature encode write context. */ void makeHeader(const search::common::FileHeaderContext &fileHeaderContext); void updateHeader(); - - /** - * Read header, using temporary feature decode context. - */ - uint32_t readHeader(const vespalib::string &name); }; @@ -223,7 +174,6 @@ class ZcPostingSeqWrite : public Zc4PostingSeqWrite { public: ZcPostingSeqWrite(index::PostingListCountFileSeqWrite *countFile); - void flushWordNoSkip() override; }; } diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.h b/searchlib/src/vespa/searchlib/index/docidandfeatures.h index 91a500495cc..a9329c9fa01 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.h +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.h @@ -183,14 +183,6 @@ public: void setRaw(bool raw) { _raw = raw; } bool getRaw() const { return _raw; } - - /** - * Append features from a single field to a field collection. - * - * @param rhs features for a single field - * @param localFieldId local field id for the field - */ - void append(const DocIdAndFeatures &rhs, uint32_t localFieldId); }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index f1127a3f554..441fe12c383 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,17 +1,17 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_memoryindex OBJECT SOURCES - compact_document_words_store.cpp - dictionary.cpp - documentinverter.cpp - document_remover.cpp - featurestore.cpp - fieldinverter.cpp - memoryfieldindex.cpp - memoryindex.cpp - ordereddocumentinserter.cpp - postingiterator.cpp - urlfieldinverter.cpp - wordstore.cpp + compact_words_store.cpp + document_inverter.cpp + feature_store.cpp + field_index.cpp + field_index_collection.cpp + field_index_remover.cpp + field_inverter.cpp + memory_index.cpp + ordered_field_index_inserter.cpp + posting_iterator.cpp + url_field_inverter.cpp + word_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp index ec8fab1991b..27282282c11 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp @@ -1,16 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compact_document_words_store.h" +#include "compact_words_store.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.compact_document_words_store"); +LOG_SETUP(".memoryindex.compact_words_store"); -namespace search { -namespace memoryindex { +namespace search::memoryindex { -typedef CompactDocumentWordsStore::Builder Builder; +using Builder = CompactWordsStore::Builder; namespace { @@ -37,36 +36,36 @@ serialize(const Builder &builder, uint32_t *begin) } -CompactDocumentWordsStore::Builder::Builder(uint32_t docId_) +CompactWordsStore::Builder::Builder(uint32_t docId_) : _docId(docId_), _words() { } -CompactDocumentWordsStore::Builder::~Builder() { } +CompactWordsStore::Builder::~Builder() { } -CompactDocumentWordsStore::Builder & -CompactDocumentWordsStore::Builder::insert(datastore::EntryRef wordRef) +CompactWordsStore::Builder & +CompactWordsStore::Builder::insert(datastore::EntryRef wordRef) { _words.push_back(wordRef); return *this; } inline void -CompactDocumentWordsStore::Iterator::nextWord() +CompactWordsStore::Iterator::nextWord() { _wordRef = *_buf++; _remainingWords--; } -CompactDocumentWordsStore::Iterator::Iterator() - : _buf(NULL), +CompactWordsStore::Iterator::Iterator() + : _buf(nullptr), _remainingWords(0), _wordRef(0), _valid(false) { } -CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) +CompactWordsStore::Iterator::Iterator(const uint32_t *buf) : _buf(buf), _remainingWords(0), _wordRef(0), @@ -80,8 +79,8 @@ CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) } } -CompactDocumentWordsStore::Iterator & -CompactDocumentWordsStore::Iterator::operator++() +CompactWordsStore::Iterator & +CompactWordsStore::Iterator::operator++() { if (_remainingWords > 0) { nextWord(); @@ -91,7 +90,7 @@ CompactDocumentWordsStore::Iterator::operator++() return *this; } -CompactDocumentWordsStore::Store::Store() +CompactWordsStore::Store::Store() : _store(), _type(1, MIN_BUFFER_ARRAYS, @@ -102,13 +101,13 @@ CompactDocumentWordsStore::Store::Store() _store.initActiveBuffers(); } -CompactDocumentWordsStore::Store::~Store() +CompactWordsStore::Store::~Store() { _store.dropBuffers(); } datastore::EntryRef -CompactDocumentWordsStore::Store::insert(const Builder &builder) +CompactWordsStore::Store::insert(const Builder &builder) { size_t serializedSize = getSerializedSize(builder); auto result = _store.rawAllocator<uint32_t>(_typeId).alloc(serializedSize); @@ -119,26 +118,26 @@ CompactDocumentWordsStore::Store::insert(const Builder &builder) return result.ref; } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::Store::get(datastore::EntryRef ref) const +CompactWordsStore::Iterator +CompactWordsStore::Store::get(datastore::EntryRef wordRef) const { - RefType internalRef(ref); + RefType internalRef(wordRef); const uint32_t *buf = _store.getEntry<uint32_t>(internalRef); return Iterator(buf); } -CompactDocumentWordsStore::CompactDocumentWordsStore() +CompactWordsStore::CompactWordsStore() : _docs(), _wordsStore() { } -CompactDocumentWordsStore::~CompactDocumentWordsStore() { } +CompactWordsStore::~CompactWordsStore() { } void -CompactDocumentWordsStore::insert(const Builder &builder) +CompactWordsStore::insert(const Builder &builder) { - datastore::EntryRef ref = _wordsStore.insert(builder); - auto insres = _docs.insert(std::make_pair(builder.docId(), ref)); + datastore::EntryRef wordRef = _wordsStore.insert(builder); + auto insres = _docs.insert(std::make_pair(builder.docId(), wordRef)); if (!insres.second) { LOG(error, "Failed inserting remove info for docid %u", builder.docId()); @@ -147,13 +146,13 @@ CompactDocumentWordsStore::insert(const Builder &builder) } void -CompactDocumentWordsStore::remove(uint32_t docId) +CompactWordsStore::remove(uint32_t docId) { _docs.erase(docId); } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::get(uint32_t docId) const +CompactWordsStore::Iterator +CompactWordsStore::get(uint32_t docId) const { auto itr = _docs.find(docId); if (itr != _docs.end()) { @@ -163,7 +162,7 @@ CompactDocumentWordsStore::get(uint32_t docId) const } MemoryUsage -CompactDocumentWordsStore::getMemoryUsage() const +CompactWordsStore::getMemoryUsage() const { MemoryUsage usage; usage.incAllocatedBytes(_docs.getMemoryConsumption()); @@ -173,6 +172,5 @@ CompactDocumentWordsStore::getMemoryUsage() const } -} // namespace memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h index d3bb2220dc5..2fc6ec8d5bb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h @@ -10,21 +10,19 @@ namespace search::memoryindex { /** - * Class used to store the {wordRef, fieldId, docId} tuples that are inserted - * into the memory index dictionary. These tuples are later used when removing - * all remains of a document from the posting lists of the dictionary. + * Class used to store the {wordRef, docId} tuples that are inserted into a FieldIndex and its posting lists. + * + * These tuples are later used when removing all remains of a document from the posting lists in that index. */ -class CompactDocumentWordsStore -{ +class CompactWordsStore { public: /** - * Builder used to collect all wordRefs for a field. + * Builder used to collect all words (as wordRefs) for a docId in a field. */ - class Builder - { + class Builder { public: - typedef std::unique_ptr<Builder> UP; - typedef vespalib::Array<datastore::EntryRef> WordRefVector; + using UP = std::unique_ptr<Builder>; + using WordRefVector = vespalib::Array<datastore::EntryRef>; private: uint32_t _docId; @@ -39,10 +37,9 @@ public: }; /** - * Iterator over all {wordRef, fieldId} pairs for a document. + * Iterator over all words (as wordRefs) for a docId in a field. */ - class Iterator - { + class Iterator { private: const uint32_t *_buf; uint32_t _remainingWords; @@ -61,13 +58,12 @@ public: }; /** - * Store for all {wordRef, fieldId} pairs among all documents. + * Store for all unique words (as wordRefs) among all documents. */ - class Store - { + class Store { public: - typedef datastore::DataStoreT<datastore::EntryRefT<22> > DataStoreType; - typedef DataStoreType::RefType RefType; + using DataStoreType = datastore::DataStoreT<datastore::EntryRefT<22>>; + using RefType = DataStoreType::RefType; private: DataStoreType _store; @@ -78,19 +74,19 @@ public: Store(); ~Store(); datastore::EntryRef insert(const Builder &builder); - Iterator get(datastore::EntryRef ref) const; + Iterator get(datastore::EntryRef wordRef) const; MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } }; - typedef vespalib::hash_map<uint32_t, datastore::EntryRef> DocumentWordsMap; + using DocumentWordsMap = vespalib::hash_map<uint32_t, datastore::EntryRef>; private: DocumentWordsMap _docs; Store _wordsStore; public: - CompactDocumentWordsStore(); - ~CompactDocumentWordsStore(); + CompactWordsStore(); + ~CompactWordsStore(); void insert(const Builder &builder); void remove(uint32_t docId); Iterator get(uint32_t docId) const; diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 08d77fa8dce..a468428e21f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -1,22 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" -#include "fieldinverter.h" -#include "urlfieldinverter.h" -#include "dictionary.h" -#include "ordereddocumentinserter.h" -#include <vespa/document/datatype/urldatatype.h> +#include "document_inverter.h" +#include "field_index_collection.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include "url_field_inverter.h" #include <vespa/document/annotation/alternatespanlist.h> -#include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> -#include <vespa/vespalib/text/lowercase.h> -#include <vespa/searchlib/common/sort.h> +#include <vespa/document/datatype/urldatatype.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> -#include <vespa/log/log.h> +#include <vespa/searchlib/common/sort.h> +#include <vespa/searchlib/util/url.h> +#include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> +#include <stdexcept> -LOG_SETUP(".memoryindex.documentinverter"); +#include <vespa/log/log.h> +LOG_SETUP(".memoryindex.document_inverter"); namespace search::memoryindex { @@ -39,7 +39,6 @@ using index::DocIdAndPosOccFeatures; using index::Schema; using search::util::URL; - DocumentInverter::DocumentInverter(const Schema &schema, ISequencedTaskExecutor &invertThreads, ISequencedTaskExecutor &pushThreads) @@ -74,14 +73,12 @@ DocumentInverter::DocumentInverter(const Schema &schema, } } - DocumentInverter::~DocumentInverter() { _invertThreads.sync(); _pushThreads.sync(); } - void DocumentInverter::addFieldPath(const document::DocumentType &docType, uint32_t fieldId) @@ -95,14 +92,14 @@ DocumentInverter::addFieldPath(const document::DocumentType &docType, _schema.getIndexField(fieldId).getName().c_str(), docType.getName().c_str()); } else { - fp.reset(new Field(docType.getField(_schema.getIndexField(fieldId).getName()))); + fp = std::make_unique<Field>(docType.getField(_schema.getIndexField(fieldId).getName())); } _indexedFieldPaths[fieldId] = std::move(fp); } - -void DocumentInverter::buildFieldPath(const document::DocumentType &docType, - const document::DataType *dataType) +void +DocumentInverter::buildFieldPath(const document::DocumentType &docType, + const document::DataType *dataType) { _indexedFieldPaths.clear(); _indexedFieldPaths.resize(_schema.getNumIndexFields()); @@ -115,7 +112,6 @@ void DocumentInverter::buildFieldPath(const document::DocumentType &docType, _dataType = dataType; } - void DocumentInverter::invertDocument(uint32_t docId, const Document &doc) { @@ -154,7 +150,6 @@ DocumentInverter::invertDocument(uint32_t docId, const Document &doc) } } - void DocumentInverter::removeDocument(uint32_t docId) { @@ -175,18 +170,16 @@ DocumentInverter::removeDocument(uint32_t docId) } } - void -DocumentInverter::pushDocuments(Dictionary &dict, - const std::shared_ptr<IDestructorCallback> & - onWriteDone) +DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, + const std::shared_ptr<IDestructorCallback> &onWriteDone) { - auto indexFieldIterator = dict.getFieldIndexes().begin(); + auto indexFieldIterator = fieldIndexes.getFieldIndexes().begin(); uint32_t fieldId = 0; for (auto &inverter : _inverters) { - MemoryFieldIndex &fieldIndex(**indexFieldIterator); - DocumentRemover &remover(fieldIndex.getDocumentRemover()); - OrderedDocumentInserter &inserter(fieldIndex.getInserter()); + FieldIndex &fieldIndex(**indexFieldIterator); + FieldIndexRemover &remover(fieldIndex.getDocumentRemover()); + OrderedFieldIndexInserter &inserter(fieldIndex.getInserter()); _pushThreads.execute(fieldId, [inverter(inverter.get()), &remover, &inserter, &fieldIndex, onWriteDone]() diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index e92c0810ea7..5c2d9cc84ed 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -2,7 +2,7 @@ #pragma once -#include "i_document_remove_listener.h" +#include "i_field_index_remove_listener.h" #include <vespa/searchlib/index/schema_index_fields.h> namespace document { @@ -22,10 +22,9 @@ namespace search::memoryindex { class FieldInverter; class UrlFieldInverter; -class Dictionary; +class FieldIndexCollection; -class DocumentInverter -{ +class DocumentInverter { private: DocumentInverter(const DocumentInverter &) = delete; DocumentInverter &operator=(const DocumentInverter &) = delete; @@ -37,9 +36,8 @@ private: void invertNormalDocTextField(size_t fieldId, const document::FieldValue &field); void invertNormalDocUriField(const index::UriField &handle, const document::FieldValue &field); - //typedef document::FieldPath FieldPath; - typedef document::Field FieldPath; - typedef std::vector<std::unique_ptr<FieldPath> > IndexedFieldPaths; + using FieldPath = document::Field; + using IndexedFieldPaths = std::vector<std::unique_ptr<FieldPath>>; IndexedFieldPaths _indexedFieldPaths; const document::DataType * _dataType; @@ -70,11 +68,9 @@ public: ~DocumentInverter(); /** - * Push inverted documents to memory index structure. - * - * @param dict dictionary + * Push inverted documents to memory field indexes. */ - void pushDocuments(Dictionary &dict, const std::shared_ptr<IDestructorCallback> &onWriteDone); + void pushDocuments(FieldIndexCollection &fieldIndexes, const std::shared_ptr<IDestructorCallback> &onWriteDone); /** * Invert a document. diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index 9fdb23876d8..974fcc01c36 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "featurestore.h" +#include "feature_store.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/datastore/datastore.hpp> @@ -26,7 +26,6 @@ FeatureStore::writeFeatures(uint32_t packedIndex, const DocIdAndFeatures &featur return oldOffset; } - datastore::EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { @@ -43,7 +42,6 @@ FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) return result.ref; } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) { @@ -58,7 +56,6 @@ FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) return std::make_pair(ref, bitLen); } - datastore::EntryRef FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) { @@ -70,12 +67,11 @@ FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) return newRef; } - FeatureStore::FeatureStore(const Schema &schema) : _store(), - _f(NULL), + _f(nullptr), _fctx(_f), - _d(NULL), + _d(nullptr), _fieldsParams(), _schema(schema), _type(RefType::align(1u), MIN_BUFFER_ARRAYS, @@ -88,20 +84,18 @@ FeatureStore::FeatureStore(const Schema &schema) _fieldsParams.resize(_schema.getNumIndexFields()); SchemaUtil::IndexIterator it(_schema); - for(; it.isValid(); ++it) { + for (; it.isValid(); ++it) { _fieldsParams[it.getIndex()].setSchemaParams(_schema, it.getIndex()); } _store.addType(&_type); _store.initActiveBuffers(); } - FeatureStore::~FeatureStore() { _store.dropBuffers(); } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features) { @@ -111,8 +105,6 @@ FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features return addFeatures(oldOffset, newOffset); } - - void FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAndFeatures &features) { @@ -121,7 +113,6 @@ FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAn _d.readFeatures(features); } - size_t FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) { @@ -135,7 +126,6 @@ FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) return bitLen; } - datastore::EntryRef FeatureStore::moveFeatures(uint32_t packedIndex, datastore::EntryRef ref) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index f3c2ad6cd03..94d44eaf44d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -9,20 +9,18 @@ namespace search::memoryindex { -class FeatureStore -{ +class FeatureStore { public: - typedef datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2> > DataStoreType; - typedef DataStoreType::RefType RefType; - typedef bitcompression::EG2PosOccEncodeContext<true> EncodeContext; - typedef bitcompression::EG2PosOccDecodeContextCooked<true> - DecodeContextCooked; - typedef vespalib::GenerationHandler::generation_t generation_t; + using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; + using RefType = DataStoreType::RefType; + using EncodeContext = bitcompression::EG2PosOccEncodeContext<true>; + using DecodeContextCooked = bitcompression::EG2PosOccDecodeContextCooked<true>; + using generation_t = vespalib::GenerationHandler::generation_t; private: - typedef index::Schema Schema; - typedef index::DocIdAndFeatures DocIdAndFeatures; - typedef bitcompression::PosOccFieldsParams PosOccFieldsParams; + using Schema = index::Schema; + using DocIdAndFeatures = index::DocIdAndFeatures; + using PosOccFieldsParams = bitcompression::PosOccFieldsParams; static const uint32_t DECODE_SAFETY = 16; @@ -123,9 +121,7 @@ public: * @param packedIndex The field or field collection owning features * @param decoder The feature decoder */ - void - setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const - { + void setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const { decoder._fieldsParams = &_fieldsParams[packedIndex]; } @@ -136,9 +132,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { const uint8_t * bits = getBits(ref); decoder.setByteCompr(bits); uint32_t bufferId = RefType(ref).bufferId(); @@ -156,9 +150,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { decoder.setByteCompr(getBits(ref)); } @@ -170,8 +162,7 @@ public: * @param ref Reference to stored features * @return size of features in bits */ - size_t - bitSize(uint32_t packedIndex, datastore::EntryRef ref); + size_t bitSize(uint32_t packedIndex, datastore::EntryRef ref); /** * Get byte address of stored features diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 4e02c3cb09d..7d10895c32f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "memoryfieldindex.h" -#include "ordereddocumentinserter.h" +#include "field_index.h" +#include "ordered_field_index_inserter.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -23,13 +23,13 @@ namespace search::memoryindex { using datastore::EntryRef; vespalib::asciistream & -operator<<(vespalib::asciistream & os, const MemoryFieldIndex::WordKey & rhs) +operator<<(vespalib::asciistream & os, const FieldIndex::WordKey & rhs) { os << "wr(" << rhs._wordRef.ref() << ")"; return os; } -MemoryFieldIndex::MemoryFieldIndex(const Schema & schema, uint32_t fieldId) +FieldIndex::FieldIndex(const Schema & schema, uint32_t fieldId) : _wordStore(), _numUniqueWords(0), _generationHandler(), @@ -38,10 +38,10 @@ MemoryFieldIndex::MemoryFieldIndex(const Schema & schema, uint32_t fieldId) _featureStore(schema), _fieldId(fieldId), _remover(_wordStore), - _inserter(std::make_unique<OrderedDocumentInserter>(*this)) + _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)) { } -MemoryFieldIndex::~MemoryFieldIndex() +FieldIndex::~FieldIndex() { _postingListStore.disableFreeLists(); _postingListStore.disableElemHoldList(); @@ -68,8 +68,8 @@ MemoryFieldIndex::~MemoryFieldIndex() trimHoldLists(); } -MemoryFieldIndex::PostingList::Iterator -MemoryFieldIndex::find(const vespalib::stringref word) const +FieldIndex::PostingList::Iterator +FieldIndex::find(const vespalib::stringref word) const { DictionaryTree::Iterator itr = _dict.find(WordKey(EntryRef()), KeyComp(_wordStore, word)); if (itr.valid()) { @@ -78,35 +78,35 @@ MemoryFieldIndex::find(const vespalib::stringref word) const return PostingList::Iterator(); } -MemoryFieldIndex::PostingList::ConstIterator -MemoryFieldIndex::findFrozen(const vespalib::stringref word) const +FieldIndex::PostingList::ConstIterator +FieldIndex::findFrozen(const vespalib::stringref word) const { - DictionaryTree::ConstIterator itr = _dict.getFrozenView().find(WordKey(EntryRef()), KeyComp(_wordStore, word)); + auto itr = _dict.getFrozenView().find(WordKey(EntryRef()), KeyComp(_wordStore, word)); if (itr.valid()) { return _postingListStore.beginFrozen(EntryRef(itr.getData())); } return PostingList::Iterator(); } - void -MemoryFieldIndex::compactFeatures() +FieldIndex::compactFeatures() { std::vector<uint32_t> toHold; toHold = _featureStore.startCompact(); - DictionaryTree::Iterator itr(_dict.begin()); + auto itr = _dict.begin(); uint32_t packedIndex = _fieldId; for (; itr.valid(); ++itr) { PostingListStore::RefType pidx(EntryRef(itr.getData())); - if (!pidx.valid()) + if (!pidx.valid()) { continue; + } uint32_t clusterSize = _postingListStore.getClusterSize(pidx); if (clusterSize == 0) { const PostingList *tree = _postingListStore.getTreeEntry(pidx); - PostingList::Iterator it(tree->begin(_postingListStore.getAllocator())); - for (; it.valid(); ++it) { - EntryRef oldFeatures(it.getData()); + auto pitr = tree->begin(_postingListStore.getAllocator()); + for (; pitr.valid(); ++pitr) { + EntryRef oldFeatures(pitr.getData()); // Filter on which buffers to move features from when // performing incremental compaction. @@ -117,7 +117,7 @@ MemoryFieldIndex::compactFeatures() std::atomic_thread_fence(std::memory_order_release); // Ugly, ugly due to const_cast in iterator - it.writeData(newFeatures.ref()); + pitr.writeData(newFeatures.ref()); } } else { const PostingListKeyDataType *shortArray = _postingListStore.getKeyDataEntry(pidx, clusterSize); @@ -139,31 +139,32 @@ MemoryFieldIndex::compactFeatures() } } } - typedef GenerationHandler::generation_t generation_t; + using generation_t = GenerationHandler::generation_t; _featureStore.finishCompact(toHold); generation_t generation = _generationHandler.getCurrentGeneration(); _featureStore.transferHoldLists(generation); } void -MemoryFieldIndex::dump(search::index::IndexBuilder & indexBuilder) +FieldIndex::dump(search::index::IndexBuilder & indexBuilder) { vespalib::stringref word; - FeatureStore::DecodeContextCooked decoder(NULL); + FeatureStore::DecodeContextCooked decoder(nullptr); DocIdAndFeatures features; vespalib::Array<uint32_t> wordMap(_numUniqueWords + 1, 0); _featureStore.setupForField(_fieldId, decoder); - for (DictionaryTree::Iterator itr = _dict.begin(); itr.valid(); ++itr) { + for (auto itr = _dict.begin(); itr.valid(); ++itr) { const WordKey & wk = itr.getKey(); PostingListStore::RefType plist(EntryRef(itr.getData())); word = _wordStore.getWord(wk._wordRef); - if (!plist.valid()) + if (!plist.valid()) { continue; + } indexBuilder.startWord(word); uint32_t clusterSize = _postingListStore.getClusterSize(plist); if (clusterSize == 0) { const PostingList *tree = _postingListStore.getTreeEntry(plist); - PostingList::Iterator pitr = tree->begin(_postingListStore.getAllocator()); + auto pitr = tree->begin(_postingListStore.getAllocator()); assert(pitr.valid()); for (; pitr.valid(); ++pitr) { uint32_t docId = pitr.getKey(); @@ -216,9 +217,8 @@ MemoryFieldIndex::dump(search::index::IndexBuilder & indexBuilder) } } - MemoryUsage -MemoryFieldIndex::getMemoryUsage() const +FieldIndex::getMemoryUsage() const { MemoryUsage usage; usage.merge(_wordStore.getMemoryUsage()); @@ -229,83 +229,83 @@ MemoryFieldIndex::getMemoryUsage() const return usage; } -} // namespace search::memoryindex +} namespace search::btree { template -class BTreeNodeDataWrap<memoryindex::MemoryFieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; +class BTreeNodeDataWrap<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeNodeT<memoryindex::MemoryFieldIndex::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>; +class BTreeNodeT<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>; #if 0 template -class BTreeNodeT<memoryindex::MemoryFieldIndex::WordKey, +class BTreeNodeT<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; #endif template -class BTreeNodeTT<memoryindex::MemoryFieldIndex::WordKey, +class BTreeNodeTT<memoryindex::FieldIndex::WordKey, datastore::EntryRef, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; template -class BTreeNodeTT<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeNodeTT<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeInternalNode<memoryindex::MemoryFieldIndex::WordKey, +class BTreeInternalNode<memoryindex::FieldIndex::WordKey, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; template -class BTreeLeafNode<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeLeafNode<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeNodeStore<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeNodeStore<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeIterator<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeIterator<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, - const memoryindex::MemoryFieldIndex::KeyComp, + const memoryindex::FieldIndex::KeyComp, BTreeDefaultTraits>; template -class BTree<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTree<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, - const memoryindex::MemoryFieldIndex::KeyComp, + const memoryindex::FieldIndex::KeyComp, BTreeDefaultTraits>; template -class BTreeRoot<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeRoot<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, - const memoryindex::MemoryFieldIndex::KeyComp, + const memoryindex::FieldIndex::KeyComp, BTreeDefaultTraits>; template -class BTreeRootBase<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeRootBase<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; template -class BTreeNodeAllocator<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeNodeAllocator<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index 5b4ab7eb4fd..3b0675b5fdf 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -2,12 +2,12 @@ #pragma once -#include "featurestore.h" -#include "wordstore.h" -#include "document_remover.h" -#include <vespa/searchlib/btree/btreeroot.h> +#include "feature_store.h" +#include "field_index_remover.h" +#include "word_store.h" #include <vespa/searchlib/btree/btree.h> #include <vespa/searchlib/btree/btreenodeallocator.h> +#include <vespa/searchlib/btree/btreeroot.h> #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/indexbuilder.h> @@ -16,20 +16,30 @@ namespace search::memoryindex { -class OrderedDocumentInserter; -/* +class OrderedFieldIndexInserter; + +/** * Memory index for a single field. + * + * It consists of the following components: + * - WordStore containing all unique words in this field (across all documents). + * - B-Tree dictionary that maps from unique word (32-bit ref) -> posting list (32-bit ref). + * - B-Tree posting lists that maps from document id (32-bit) -> features (32-bit ref). + * - BTreeStore containing all the posting lists. + * - FeatureStore containing information on where a (word, document) pair matched this field. + * This information is unpacked and used during ranking. + * + * Elements in the three stores are accessed using 32-bit references / handles. */ -class MemoryFieldIndex { +class FieldIndex { public: - typedef btree::BTreeRoot<uint32_t, uint32_t, search::btree::NoAggregated> - PostingList; // docid -> feature ref - typedef btree::BTreeStore<uint32_t, uint32_t, - search::btree::NoAggregated, - std::less<uint32_t>, - btree::BTreeDefaultTraits> PostingListStore; - typedef PostingListStore::KeyDataType PostingListKeyDataType; - + // Mapping from docid -> feature ref + using PostingList = btree::BTreeRoot<uint32_t, uint32_t, search::btree::NoAggregated>; + using PostingListStore = btree::BTreeStore<uint32_t, uint32_t, + search::btree::NoAggregated, + std::less<uint32_t>, + btree::BTreeDefaultTraits>; + using PostingListKeyDataType = PostingListStore::KeyDataType; struct WordKey { datastore::EntryRef _wordRef; @@ -46,9 +56,7 @@ public: const WordStore &_wordStore; const vespalib::stringref _word; - const char * - getWord(datastore::EntryRef wordRef) const - { + const char *getWord(datastore::EntryRef wordRef) const { if (wordRef.valid()) { return _wordStore.getWord(wordRef); } @@ -61,20 +69,18 @@ public: _word(word) { } - bool - operator()(const WordKey & lhs, const WordKey & rhs) const - { + bool operator()(const WordKey & lhs, const WordKey & rhs) const { int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef)); return cmpres < 0; } }; - typedef uint32_t PostingListPtr; - typedef btree::BTree<WordKey, PostingListPtr, - search::btree::NoAggregated, - const KeyComp> DictionaryTree; + using PostingListPtr = uint32_t; + using DictionaryTree = btree::BTree<WordKey, PostingListPtr, + search::btree::NoAggregated, + const KeyComp>; private: - typedef vespalib::GenerationHandler GenerationHandler; + using GenerationHandler = vespalib::GenerationHandler; WordStore _wordStore; uint64_t _numUniqueWords; @@ -83,8 +89,8 @@ private: PostingListStore _postingListStore; FeatureStore _featureStore; uint32_t _fieldId; - DocumentRemover _remover; - std::unique_ptr<OrderedDocumentInserter> _inserter; + FieldIndexRemover _remover; + std::unique_ptr<OrderedFieldIndexInserter> _inserter; public: datastore::EntryRef addWord(const vespalib::stringref word) { @@ -92,14 +98,12 @@ public: return _wordStore.addWord(word); } - datastore::EntryRef - addFeatures(const index::DocIdAndFeatures &features) - { + datastore::EntryRef addFeatures(const index::DocIdAndFeatures &features) { return _featureStore.addFeatures(_fieldId, features).first; } - MemoryFieldIndex(const index::Schema &schema, uint32_t fieldId); - ~MemoryFieldIndex(); + FieldIndex(const index::Schema &schema, uint32_t fieldId); + ~FieldIndex(); PostingList::Iterator find(const vespalib::stringref word) const; PostingList::ConstIterator @@ -108,7 +112,7 @@ public: uint64_t getNumUniqueWords() const { return _numUniqueWords; } const FeatureStore & getFeatureStore() const { return _featureStore; } const WordStore &getWordStore() const { return _wordStore; } - OrderedDocumentInserter &getInserter() const { return *_inserter; } + OrderedFieldIndexInserter &getInserter() const { return *_inserter; } private: void freeze() { @@ -116,9 +120,7 @@ private: _dict.getAllocator().freeze(); } - void - trimHoldLists() - { + void trimHoldLists() { GenerationHandler::generation_t usedGen = _generationHandler.getFirstUsedGeneration(); _postingListStore.trimHoldLists(usedGen); @@ -126,9 +128,7 @@ private: _featureStore.trimHoldLists(usedGen); } - void - transferHoldLists() - { + void transferHoldLists() { GenerationHandler::generation_t generation = _generationHandler.getCurrentGeneration(); _postingListStore.transferHoldLists(generation); @@ -136,9 +136,7 @@ private: _featureStore.transferHoldLists(generation); } - void - incGeneration() - { + void incGeneration() { _generationHandler.incGeneration(); } @@ -153,27 +151,11 @@ public: void dump(search::index::IndexBuilder & indexBuilder); MemoryUsage getMemoryUsage() const; + DictionaryTree &getDictionaryTree() { return _dict; } + PostingListStore &getPostingListStore() { return _postingListStore; } + FieldIndexRemover &getDocumentRemover() { return _remover; } - DictionaryTree & - getDictionaryTree() - { - return _dict; - } - - PostingListStore & - getPostingListStore() - { - return _postingListStore; - } - - DocumentRemover & - getDocumentRemover() - { - return _remover; - } - - void commit() - { + void commit() { _remover.flush(); freeze(); transferHoldLists(); @@ -187,80 +169,80 @@ public: namespace search::btree { extern template -class BTreeNodeDataWrap<memoryindex::MemoryFieldIndex::WordKey, +class BTreeNodeDataWrap<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeNodeT<memoryindex::MemoryFieldIndex::WordKey, +class BTreeNodeT<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::INTERNAL_SLOTS>; #if 0 extern template -class BTreeNodeT<memoryindex::MemoryFieldIndex::WordKey, +class BTreeNodeT<memoryindex::FieldIndex::WordKey, BTreeDefaultTraits::LEAF_SLOTS>; #endif extern template -class BTreeNodeTT<memoryindex::MemoryFieldIndex::WordKey, +class BTreeNodeTT<memoryindex::FieldIndex::WordKey, datastore::EntryRef, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; extern template -class BTreeNodeTT<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeNodeTT<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeInternalNode<memoryindex::MemoryFieldIndex::WordKey, +class BTreeInternalNode<memoryindex::FieldIndex::WordKey, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS>; extern template -class BTreeLeafNode<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeLeafNode<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeNodeStore<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeNodeStore<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeIterator<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeIterator<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, - const memoryindex::MemoryFieldIndex::KeyComp, + const memoryindex::FieldIndex::KeyComp, BTreeDefaultTraits>; extern template -class BTree<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTree<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, - const memoryindex::MemoryFieldIndex::KeyComp, + const memoryindex::FieldIndex::KeyComp, BTreeDefaultTraits>; extern template -class BTreeRoot<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeRoot<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, - const memoryindex::MemoryFieldIndex::KeyComp, + const memoryindex::FieldIndex::KeyComp, BTreeDefaultTraits>; extern template -class BTreeRootBase<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeRootBase<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; extern template -class BTreeNodeAllocator<memoryindex::MemoryFieldIndex::WordKey, - memoryindex::MemoryFieldIndex::PostingListPtr, +class BTreeNodeAllocator<memoryindex::FieldIndex::WordKey, + memoryindex::FieldIndex::PostingListPtr, search::btree::NoAggregated, BTreeDefaultTraits::INTERNAL_SLOTS, BTreeDefaultTraits::LEAF_SLOTS>; diff --git a/searchlib/src/vespa/searchlib/memoryindex/dictionary.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index accf227b96c..27944b5fe89 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/dictionary.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dictionary.h" -#include "fieldinverter.h" +#include "field_index_collection.h" +#include "field_inverter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/btree/btreenode.hpp> @@ -15,7 +15,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.dictionary"); +LOG_SETUP(".searchlib.memoryindex.field_index_collection"); namespace search { @@ -26,23 +26,22 @@ using index::Schema; namespace memoryindex { -Dictionary::Dictionary(const Schema & schema) +FieldIndexCollection::FieldIndexCollection(const Schema & schema) : _fieldIndexes(), _numFields(schema.getNumIndexFields()) { for (uint32_t fieldId = 0; fieldId < _numFields; ++fieldId) { - auto fieldIndex = std::make_unique<MemoryFieldIndex>(schema, fieldId); + auto fieldIndex = std::make_unique<FieldIndex>(schema, fieldId); _fieldIndexes.push_back(std::move(fieldIndex)); } } -Dictionary::~Dictionary() +FieldIndexCollection::~FieldIndexCollection() { } - void -Dictionary::dump(search::index::IndexBuilder &indexBuilder) +FieldIndexCollection::dump(search::index::IndexBuilder &indexBuilder) { for (uint32_t fieldId = 0; fieldId < _numFields; ++fieldId) { indexBuilder.startField(fieldId); @@ -52,7 +51,7 @@ Dictionary::dump(search::index::IndexBuilder &indexBuilder) } MemoryUsage -Dictionary::getMemoryUsage() const +FieldIndexCollection::getMemoryUsage() const { MemoryUsage usage; for (auto &fieldIndex : _fieldIndexes) { @@ -61,7 +60,5 @@ Dictionary::getMemoryUsage() const return usage; } - -} // namespace search::memoryindex - -} // namespace search +} +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/dictionary.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index e9db79c1bd8..5c2aa6f9b2c 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/dictionary.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -2,35 +2,38 @@ #pragma once -#include "memoryfieldindex.h" +#include "field_index.h" namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class FieldInverter; -class Dictionary { +/** + * The collection of all field indexes that are part of a memory index. + * + * Provides functions to create a posting list iterator (used for searching) + * for a given word in a given field. + */ +class FieldIndexCollection { public: - using PostingList = MemoryFieldIndex::PostingList; + using PostingList = FieldIndex::PostingList; private: - typedef vespalib::GenerationHandler GenerationHandler; + using GenerationHandler = vespalib::GenerationHandler; - std::vector<std::unique_ptr<MemoryFieldIndex> > _fieldIndexes; + std::vector<std::unique_ptr<FieldIndex>> _fieldIndexes; uint32_t _numFields; public: - Dictionary(const index::Schema &schema); - ~Dictionary(); + FieldIndexCollection(const index::Schema &schema); + ~FieldIndexCollection(); PostingList::Iterator find(const vespalib::stringref word, - uint32_t fieldId) const - { + uint32_t fieldId) const { return _fieldIndexes[fieldId]->find(word); } - PostingList::ConstIterator - findFrozen(const vespalib::stringref word, uint32_t fieldId) const - { + PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t fieldId) const { return _fieldIndexes[fieldId]->findFrozen(word); } @@ -46,12 +49,11 @@ public: MemoryUsage getMemoryUsage() const; - MemoryFieldIndex *getFieldIndex(uint32_t fieldId) const { + FieldIndex *getFieldIndex(uint32_t fieldId) const { return _fieldIndexes[fieldId].get(); } - const std::vector<std::unique_ptr<MemoryFieldIndex> > & - getFieldIndexes() const { return _fieldIndexes; } + const std::vector<std::unique_ptr<FieldIndex>> &getFieldIndexes() const { return _fieldIndexes; } uint32_t getNumFields() const { return _numFields; } }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp index 6ab17d5c524..2afddf072f2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp @@ -1,16 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "document_remover.h" -#include "i_document_remove_listener.h" -#include "wordstore.h" + +#include "field_index_remover.h" +#include "i_field_index_remove_listener.h" +#include "word_store.h" #include <vespa/searchlib/common/sort.h> -namespace search { -namespace memoryindex { +namespace search::memoryindex { -typedef CompactDocumentWordsStore::Builder Builder; -typedef CompactDocumentWordsStore::Iterator Iterator; +using Builder = CompactWordsStore::Builder; +using Iterator = CompactWordsStore::Iterator; -DocumentRemover::DocumentRemover(const WordStore &wordStore) +FieldIndexRemover::FieldIndexRemover(const WordStore &wordStore) : _store(), _builder(), _wordFieldDocTuples(), @@ -18,11 +18,10 @@ DocumentRemover::DocumentRemover(const WordStore &wordStore) { } -DocumentRemover::~DocumentRemover() { -} +FieldIndexRemover::~FieldIndexRemover() = default; void -DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) +FieldIndexRemover::remove(uint32_t docId, IFieldIndexRemoveListener &listener) { Iterator itr = _store.get(docId); if (itr.valid()) { @@ -35,25 +34,24 @@ DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) } void -DocumentRemover::insert(datastore::EntryRef wordRef, uint32_t docId) +FieldIndexRemover::insert(datastore::EntryRef wordRef, uint32_t docId) { _wordFieldDocTuples.emplace_back(wordRef, docId); } - void -DocumentRemover::flush() +FieldIndexRemover::flush() { if (_wordFieldDocTuples.empty()) { return; } ShiftBasedRadixSorter<WordFieldDocTuple, WordFieldDocTuple::Radix, std::less<WordFieldDocTuple>, 24, true>:: radix_sort(WordFieldDocTuple::Radix(), std::less<WordFieldDocTuple>(), &_wordFieldDocTuples[0], _wordFieldDocTuples.size(), 16); - Builder::UP builder(new Builder(_wordFieldDocTuples[0]._docId)); + auto builder = std::make_unique<Builder>(_wordFieldDocTuples[0]._docId); for (const auto &tuple : _wordFieldDocTuples) { if (builder->docId() != tuple._docId) { _store.insert(*builder); - builder.reset(new Builder(tuple._docId)); + builder = std::make_unique<Builder>(tuple._docId); } builder->insert(tuple._wordRef); } @@ -61,6 +59,4 @@ DocumentRemover::flush() _wordFieldDocTuples.clear(); } - -} // namespace memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h index dbd5bf5d472..19b3353a27a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h @@ -1,23 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "compact_document_words_store.h" -#include "i_document_insert_listener.h" +#include "compact_words_store.h" +#include "i_field_index_insert_listener.h" -namespace search { -namespace memoryindex { +namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class WordStore; /** - * Class used to remove documents from the memory index dictionary. + * Class used to handle removal of documents from a FieldIndex. + * + * It tracks all {word, docId} tuples that are inserted into the index, + * and when removing a document, all these {word, docId} tuples are sent to the component + * that is doing the actual removal (IFieldIndexRemoveListener). */ -class DocumentRemover : public IDocumentInsertListener -{ +class FieldIndexRemover : public IFieldIndexInsertListener { private: - struct WordFieldDocTuple - { + struct WordFieldDocTuple { datastore::EntryRef _wordRef; uint32_t _docId; WordFieldDocTuple() : @@ -39,26 +40,24 @@ private: return wft._docId; } }; - }; - CompactDocumentWordsStore _store; - CompactDocumentWordsStore::Builder::UP _builder; + CompactWordsStore _store; + CompactWordsStore::Builder::UP _builder; std::vector<WordFieldDocTuple> _wordFieldDocTuples; const WordStore &_wordStore; public: - DocumentRemover(const WordStore &wordStore); - ~DocumentRemover(); - void remove(uint32_t docId, IDocumentRemoveListener &inverter); - CompactDocumentWordsStore &getStore() { return _store; } - const CompactDocumentWordsStore &getStore() const { return _store; } + FieldIndexRemover(const WordStore &wordStore); + ~FieldIndexRemover(); + void remove(uint32_t docId, IFieldIndexRemoveListener &inverter); + CompactWordsStore &getStore() { return _store; } + const CompactWordsStore &getStore() const { return _store; } - // Implements IDocumentInsertListener + // Implements IFieldIndexInsertListener void insert(datastore::EntryRef wordRef, uint32_t docId) override; void flush() override; }; -} // namespace memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index 78f33b9fae8..d19f05a98ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -1,29 +1,27 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include <vespa/document/annotation/alternatespanlist.h> +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/annotation/spantreevisitor.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/bitcompression/compression.h> +#include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/alternatespanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/document/annotation/spantreevisitor.h> - -namespace search { +#include <stdexcept> -namespace memoryindex { +namespace search::memoryindex { using document::AlternateSpanList; using document::Annotation; @@ -50,23 +48,17 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; -namespace documentinverterkludge { - -namespace linguistics { +namespace documentinverterkludge::linguistics { const vespalib::string SPANTREE_NAME("linguistics"); } -} - using namespace documentinverterkludge; -namespace -{ +namespace { -class SpanFinder : public SpanTreeVisitor -{ +class SpanFinder : public SpanTreeVisitor { public: int32_t begin_pos; int32_t end_pos; @@ -112,11 +104,12 @@ FieldInverter::processAnnotations(const StringFieldValue &value) _terms.clear(); StringFieldValue::SpanTrees spanTrees = value.getSpanTrees(); const SpanTree *tree = StringFieldValue::findTree(spanTrees, linguistics::SPANTREE_NAME); - if (tree == NULL) { + if (tree == nullptr) { /* This is wrong unless field is exact match */ const vespalib::string &text = value.getValue(); - if (text.empty()) + if (text.empty()) { return; + } uint32_t wordRef = saveWord(text); if (wordRef != 0u) { add(wordRef); @@ -166,7 +159,6 @@ FieldInverter::processAnnotations(const StringFieldValue &value) } } - void FieldInverter::reset() { @@ -229,14 +221,12 @@ FieldInverter::sortWords() } } - void FieldInverter::startElement(int32_t weight) { _elems.push_back(ElemInfo(weight)); // Fill in length later } - void FieldInverter::endElement() { @@ -251,8 +241,9 @@ FieldInverter::saveWord(const vespalib::stringref word) const size_t wordsSize = _words.size(); // assert((wordsSize & 3) == 0); // Check alignment size_t len = word.size(); - if (len == 0) + if (len == 0) { return 0u; + } const size_t fullyPaddedSize = (wordsSize + 4 + len + 1 + 3) & ~3; _words.reserve(vespalib::roundUp2inN(fullyPaddedSize)); @@ -270,17 +261,15 @@ FieldInverter::saveWord(const vespalib::stringref word) return wordRef; } - uint32_t FieldInverter::saveWord(const document::FieldValue &fv) { assert(fv.getClass().id() == StringFieldValue::classId); - typedef std::pair<const char*, size_t> RawRef; + using RawRef = std::pair<const char*, size_t>; RawRef sRef = fv.getAsRaw(); return saveWord(vespalib::stringref(sRef.first, sRef.second)); } - void FieldInverter::remove(const vespalib::stringref word, uint32_t docId) { @@ -289,7 +278,6 @@ FieldInverter::remove(const vespalib::stringref word, uint32_t docId) _positions.emplace_back(wordRef, docId); } - void FieldInverter::processNormalDocTextField(const StringFieldValue &field) { @@ -298,7 +286,6 @@ FieldInverter::processNormalDocTextField(const StringFieldValue &field) endElement(); } - void FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) { @@ -307,15 +294,13 @@ FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) for (;el < ele; ++el) { const FieldValue &elfv = field[el]; assert(elfv.getClass().id() == StringFieldValue::classId); - const StringFieldValue &element = - static_cast<const StringFieldValue &>(elfv); + const auto &element = static_cast<const StringFieldValue &>(elfv); startElement(1); processAnnotations(element); endElement(); } } - void FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue &field) { @@ -324,7 +309,7 @@ FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue const FieldValue &xweight = *el.second; assert(key.getClass().id() == StringFieldValue::classId); assert(xweight.getClass().id() == IntFieldValue::classId); - const StringFieldValue &element = static_cast<const StringFieldValue &>(key); + const auto &element = static_cast<const StringFieldValue &>(key); int32_t weight = xweight.getAsInt(); startElement(weight); processAnnotations(element); @@ -332,7 +317,6 @@ FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue } } - FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) : _fieldId(fieldId), _elem(0u), @@ -353,7 +337,6 @@ FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) { } - void FieldInverter::abortPendingDoc(uint32_t docId) { @@ -366,7 +349,6 @@ FieldInverter::abortPendingDoc(uint32_t docId) } } - void FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, @@ -374,8 +356,9 @@ FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, { assert(nextTrimIdx >= srcIdx); uint32_t size = nextTrimIdx - srcIdx; - if (size == 0) + if (size == 0) { return; + } assert(dstIdx < srcIdx); assert(srcIdx < _positions.size()); assert(srcIdx + size <= _positions.size()); @@ -390,7 +373,6 @@ FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, dstIdx += size; } - void FieldInverter::trimAbortedDocs() { @@ -413,7 +395,6 @@ FieldInverter::trimAbortedDocs() _abortedDocs.clear(); } - void FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) { @@ -424,7 +405,6 @@ FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) endDoc(); } - void FieldInverter::invertNormalDocTextField(const FieldValue &val) { @@ -440,7 +420,7 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) break; case CollectionType::WEIGHTEDSET: if (cInfo.id() == WeightedSetFieldValue::classId) { - const WeightedSetFieldValue &wset = static_cast<const WeightedSetFieldValue &>(val); + const auto &wset = static_cast<const WeightedSetFieldValue &>(val); if (wset.getNestedType() == *DataType::STRING) { processNormalDocWeightedSetTextField(wset); } else { @@ -452,7 +432,7 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) break; case CollectionType::ARRAY: if (cInfo.id() == ArrayFieldValue::classId) { - const ArrayFieldValue &arr = static_cast<const ArrayFieldValue&>(val); + const auto &arr = static_cast<const ArrayFieldValue&>(val); if (arr.getNestedType() == *DataType::STRING) { processNormalDocArrayTextField(arr); } else { @@ -467,7 +447,6 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) } } - namespace { struct FullRadix { @@ -479,9 +458,8 @@ struct FullRadix { } - void -FieldInverter::applyRemoves(DocumentRemover &remover) +FieldInverter::applyRemoves(FieldIndexRemover &remover) { for (auto docId : _removeDocs) { remover.remove(docId, *this); @@ -489,9 +467,8 @@ FieldInverter::applyRemoves(DocumentRemover &remover) _removeDocs.clear(); } - void -FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) +FieldInverter::pushDocuments(IOrderedFieldIndexInserter &inserter) { trimAbortedDocs(); @@ -568,8 +545,5 @@ FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) reset(); } - -} // namespace memoryindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 96108a50f77..ecf2f8d8979 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -2,31 +2,25 @@ #pragma once -#include <map> -#include <set> -#include <vespa/document/fieldvalue/document.h> +#include "i_field_index_remove_listener.h" +#include <vespa/document/annotation/span.h> #include <vespa/document/datatype/datatypes.h> -#include <limits> -#include "i_document_remove_listener.h" -#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/span.h> - -namespace search -{ +#include <vespa/searchlib/index/docidandfeatures.h> +#include <limits> +#include <map> +#include <set> -namespace memoryindex -{ +namespace search::memoryindex { -class IOrderedDocumentInserter; -class DocumentRemover; +class IOrderedFieldIndexInserter; +class FieldIndexRemover; -class FieldInverter : public IDocumentRemoveListener -{ +class FieldInverter : public IFieldIndexRemoveListener { public: - class PosInfo - { + class PosInfo { public: uint32_t _wordNum; // XXX: Initially word reference uint32_t _docId; @@ -58,7 +52,6 @@ public: { } - PosInfo(uint32_t wordRef, uint32_t docId) : _wordNum(wordRef), @@ -69,22 +62,19 @@ public: { } - bool - removed() const - { - return _elemId == _elemRemoved; - } + bool removed() const { return _elemId == _elemRemoved; } - bool - operator<(const PosInfo &rhs) const - { - if (_wordNum != rhs._wordNum) + bool operator<(const PosInfo &rhs) const { + if (_wordNum != rhs._wordNum) { return _wordNum < rhs._wordNum; - if (_docId != rhs._docId) + } + if (_docId != rhs._docId) { return _docId < rhs._docId; + } if (_elemId != rhs._elemId) { - if (removed() != rhs.removed()) + if (removed() != rhs.removed()) { return removed() && !rhs.removed(); + } return _elemId < rhs._elemId; } return _wordPos < rhs._wordPos; @@ -97,10 +87,9 @@ private: FieldInverter &operator=(const FieldInverter &) = delete; FieldInverter &operator=(const FieldInverter &&) = delete; - typedef vespalib::Array<char> WordBuffer; + using WordBuffer = vespalib::Array<char>; - class ElemInfo - { + class ElemInfo { public: int32_t _weight; uint32_t _len; @@ -111,19 +100,13 @@ private: { } - void - setLen(uint32_t len) - { - _len = len; - } + void setLen(uint32_t len) { _len = len; } }; - typedef std::vector<ElemInfo> ElemInfoVec; + using ElemInfoVec = std::vector<ElemInfo>; + using PosInfoVec = std::vector<PosInfo>; - typedef std::vector<PosInfo> PosInfoVec; - - class CompareWordRef - { + class CompareWordRef { const char *const _wordBuffer; public: @@ -132,15 +115,11 @@ private: { } - const char * - getWord(uint32_t wordRef) const - { + const char *getWord(uint32_t wordRef) const { return &_wordBuffer[static_cast<size_t>(wordRef) << 2]; } - bool - operator()(const uint32_t lhs, const uint32_t rhs) const - { + bool operator()(const uint32_t lhs, const uint32_t rhs) const { return strcmp(getWord(lhs), getWord(rhs)) < 0; } }; @@ -148,8 +127,7 @@ private: /* * Range in _positions vector used to represent a document put. */ - class PositionRange - { + class PositionRange { uint32_t _start; uint32_t _len; @@ -160,9 +138,7 @@ private: { } - bool - operator<(const PositionRange &rhs) const - { + bool operator<(const PositionRange &rhs) const { if (_start != rhs._start) { return _start < rhs._start; } @@ -189,8 +165,8 @@ private: std::vector<uint32_t> _elementWordRefs; std::vector<uint32_t> _wordRefs; - typedef std::pair<document::Span, const document::FieldValue *> SpanTerm; - typedef std::vector<SpanTerm> SpanTermVector; + using SpanTerm = std::pair<document::Span, const document::FieldValue *>; + using SpanTermVector = std::vector<SpanTerm>; SpanTermVector _terms; // info about aborted and pending documents. @@ -207,14 +183,12 @@ public: * * @param weight element weight */ - void - startElement(int32_t weight); + void startElement(int32_t weight); /** * End an element. */ - void - endElement(); + void endElement(); private: /** @@ -225,8 +199,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const vespalib::stringref word); + VESPA_DLL_LOCAL uint32_t saveWord(const vespalib::stringref word); /** * Save field value as word in word buffer. @@ -235,8 +208,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const document::FieldValue &fv); + VESPA_DLL_LOCAL uint32_t saveWord(const document::FieldValue &fv); /** * Get pointer to saved word from a word reference. @@ -245,9 +217,7 @@ private: * * @return saved word */ - const char * - getWordFromRef(uint32_t wordRef) const - { + const char *getWordFromRef(uint32_t wordRef) const { return &_words[static_cast<size_t>(wordRef) << 2]; } @@ -258,9 +228,7 @@ private: * * @return saved word */ - const char * - getWordFromNum(uint32_t wordNum) const - { + const char *getWordFromNum(uint32_t wordNum) const { return getWordFromRef(_wordRefs[wordNum]); } @@ -271,9 +239,7 @@ private: * * @return word number */ - uint32_t - getWordNum(uint32_t wordRef) const - { + uint32_t getWordNum(uint32_t wordRef) const { const char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; return *reinterpret_cast<const uint32_t *>(p); } @@ -284,9 +250,7 @@ private: * @param wordRef word reference * @param wordNum word number */ - void - updateWordNum(uint32_t wordRef, uint32_t wordNum) - { + void updateWordNum(uint32_t wordRef, uint32_t wordNum) { char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; *reinterpret_cast<uint32_t *>(p) = wordNum; } @@ -297,17 +261,12 @@ private: * * @param wordRef word reference */ - void - add(uint32_t wordRef) { + void add(uint32_t wordRef) { _positions.emplace_back(wordRef, _docId, _elem, _wpos, _elems.size() - 1); } - void - stepWordPos() - { - ++_wpos; - } + void stepWordPos() { ++_wpos; } public: VESPA_DLL_LOCAL void @@ -328,30 +287,22 @@ private: * * @return schema used by this index */ - const index::Schema & - getSchema() const - { - return _schema; - } + const index::Schema &getSchema() const { return _schema; } /** * Clear internal memory structures. */ - void - reset(); + void reset(); /** * Calculate word numbers and replace word references with word * numbers in internal memory structures. */ - void - sortWords(); + void sortWords(); - void - moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); + void moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); - void - trimAbortedDocs(); + void trimAbortedDocs(); /* * Abort a pending document that has already been inverted. @@ -359,8 +310,7 @@ private: * @param docId local id for document * */ - void - abortPendingDoc(uint32_t docId); + void abortPendingDoc(uint32_t docId); public: /** @@ -376,42 +326,31 @@ public: * * @param remover document remover */ - void - applyRemoves(DocumentRemover &remover); + void applyRemoves(FieldIndexRemover &remover); /** - * Push inverted documents to memory index structure. + * Push inverted documents to field index structure using the given inserter. * - * Temporary restriction: Currently only one document at a time is - * supported. - * - * @param inserter ordered document inserter + * Temporary restriction: Currently only one document at a time is supported. */ - void - pushDocuments(IOrderedDocumentInserter &inserter); + void pushDocuments(IOrderedFieldIndexInserter &inserter); /* * Invert a normal text field, based on annotations. */ - void - invertField(uint32_t docId, const document::FieldValue::UP &val); + void invertField(uint32_t docId, const document::FieldValue::UP &val); /* * Setup remove of word in old version of document. */ - virtual void - remove(const vespalib::stringref word, uint32_t docId) override; + virtual void remove(const vespalib::stringref word, uint32_t docId) override; - void - removeDocument(uint32_t docId) - { + void removeDocument(uint32_t docId) { abortPendingDoc(docId); _removeDocs.push_back(docId); } - void - startDoc(uint32_t docId) - { + void startDoc(uint32_t docId) { assert(_docId == 0); assert(docId != 0); abortPendingDoc(docId); @@ -421,9 +360,7 @@ public: _wpos = 0; } - void - endDoc() - { + void endDoc() { uint32_t newPosSize = static_cast<uint32_t>(_positions.size()); _pendingDocs.insert({ _docId, { _oldPosSize, newPosSize - _oldPosSize } }); @@ -431,9 +368,7 @@ public: _oldPosSize = newPosSize; } - void - addWord(const vespalib::stringref word) - { + void addWord(const vespalib::stringref word) { uint32_t wordRef = saveWord(word); if (wordRef != 0u) { add(wordRef); @@ -442,7 +377,5 @@ public: } }; -} // namespace memoryindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h deleted file mode 100644 index a37f87aed97..00000000000 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once -#include <vespa/searchlib/datastore/entryref.h> - -namespace search { -namespace memoryindex { - -/** - * Interface used to track which {wordRef, fieldId} pairs that are - * inserted into the memory index dictionary for a document. - */ -class IDocumentInsertListener -{ -public: - virtual ~IDocumentInsertListener() {} - virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; - virtual void flush() = 0; -}; - - -} // namespace memoryindex -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h new file mode 100644 index 00000000000..0aacfa53c34 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/searchlib/datastore/entryref.h> + +namespace search::memoryindex { + +/** + * Interface used to track which {wordRef, docId} pairs that are inserted into a FieldIndex. + */ +class IFieldIndexInsertListener { +public: + virtual ~IFieldIndexInsertListener() {} + + /** + * Called when a {wordRef, docId} tuple is inserted into the field index. + */ + virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; + + /** + * Called to process the set of {wordRef, docId} tuples inserted since last flush(). + */ + virtual void flush() = 0; +}; + +} + diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h index 436ee0a49e3..4419303a654 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h @@ -7,13 +7,15 @@ namespace search::memoryindex { /** - * Interface used to track which {wordRef, fieldId} pairs that are - * removed from the memory index dictionary for a document. + * Interface used to track which {word, docId} pairs that are removed from a FieldIndex. */ -class IDocumentRemoveListener -{ +class IFieldIndexRemoveListener { public: - virtual ~IDocumentRemoveListener() {} + virtual ~IFieldIndexRemoveListener() {} + + /** + * Called when a {word, docId} tuple is removed from the field index. + */ virtual void remove(const vespalib::stringref word, uint32_t docId) = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h index 9edd1eb4d3b..a1eee2e10ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h @@ -10,13 +10,14 @@ namespace search::index { class DocIdAndFeatures; } namespace search::memoryindex { /** - * Interface class for ordered document inserter. + * Interface used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class IOrderedDocumentInserter { +class IOrderedFieldIndexInserter { public: - virtual ~IOrderedDocumentInserter() {} + virtual ~IOrderedFieldIndexInserter() {} /** * Set next word to operate on. @@ -24,7 +25,7 @@ public: virtual void setNextWord(const vespalib::stringref word) = 0; /** - * Add (word, docId) tuple with given features. + * Add (word, docId) tuple with the given features. */ virtual void add(uint32_t docId, const index::DocIdAndFeatures &features) = 0; @@ -33,15 +34,13 @@ public: */ virtual void remove(uint32_t docId) = 0; - /* - * Flush pending changes to postinglist for (_word). - * - * _dItr is located at correct position. + /** + * Flush pending changes for the current word (into the underlying posting list). */ virtual void flush() = 0; - /* - * Rewind iterator, to start new pass. + /** + * Rewind to prepare for another set of (word, docId) tuples. */ virtual void rewind() = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 4c9ea41183a..3ff2d553a96 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -1,20 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "memoryindex.h" -#include "postingiterator.h" -#include "documentinverter.h" +#include "document_inverter.h" +#include "field_index_collection.h" +#include "memory_index.h" +#include "posting_iterator.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/searchlib/btree/btreenodeallocator.hpp> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.memoryindex"); +LOG_SETUP(".searchlib.memoryindex.memory_index"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; @@ -61,7 +62,7 @@ MemoryIndex::MemoryIndex(const Schema &schema, _inverter0(std::make_unique<DocumentInverter>(_schema, _invertThreads, _pushThreads)), _inverter1(std::make_unique<DocumentInverter>(_schema, _invertThreads, _pushThreads)), _inverter(_inverter0.get()), - _dictionary(std::make_unique<Dictionary>(_schema)), + _fieldIndexes(std::make_unique<FieldIndexCollection>(_schema)), _frozen(false), _maxDocId(0), // docId 0 is reserved _numDocs(0), @@ -113,11 +114,10 @@ MemoryIndex::commit(const std::shared_ptr<IDestructorCallback> &onWriteDone) { _invertThreads.sync(); // drain inverting into this inverter _pushThreads.sync(); // drain use of other inverter - _inverter->pushDocuments(*_dictionary, onWriteDone); + _inverter->pushDocuments(*_fieldIndexes, onWriteDone); flipInverter(); } - void MemoryIndex::flipInverter() { @@ -133,23 +133,22 @@ MemoryIndex::freeze() void MemoryIndex::dump(IndexBuilder &indexBuilder) { - _dictionary->dump(indexBuilder); + _fieldIndexes->dump(indexBuilder); } namespace { -class MemTermBlueprint : public queryeval::SimpleLeafBlueprint -{ +class MemTermBlueprint : public queryeval::SimpleLeafBlueprint { private: GenerationHandler::Guard _genGuard; - Dictionary::PostingList::ConstIterator _pitr; + FieldIndex::PostingList::ConstIterator _pitr; const FeatureStore &_featureStore; const uint32_t _fieldId; const bool _useBitVector; public: MemTermBlueprint(GenerationHandler::Guard &&genGuard, - Dictionary::PostingList::ConstIterator pitr, + FieldIndex::PostingList::ConstIterator pitr, const FeatureStore &featureStore, const FieldSpecBase &field, uint32_t fieldId, @@ -166,13 +165,12 @@ public: setEstimate(estimate); } - SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { - SearchIterator::UP search(new PostingIterator(_pitr, _featureStore, _fieldId, tfmda)); + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { + auto search = std::make_unique<PostingIterator>(_pitr, _featureStore, _fieldId, tfmda); if (_useBitVector) { LOG(debug, "Return BooleanMatchIteratorWrapper: fieldId(%u), docCount(%zu)", _fieldId, _pitr.size()); - return SearchIterator::UP(new BooleanMatchIteratorWrapper(std::move(search), tfmda)); + return std::make_unique<BooleanMatchIteratorWrapper>(std::move(search), tfmda); } LOG(debug, "Return PostingIterator: fieldId(%u), docCount(%zu)", _fieldId, _pitr.size()); @@ -184,37 +182,35 @@ public: /** * Determines the correct Blueprint to use. **/ -class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper -{ +class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper { private: const FieldSpec &_field; const uint32_t _fieldId; - Dictionary & _dictionary; + FieldIndexCollection &_fieldIndexes; public: CreateBlueprintVisitor(Searchable &searchable, const IRequestContext & requestContext, const FieldSpec &field, uint32_t fieldId, - Dictionary &dictionary) + FieldIndexCollection &fieldIndexes) : CreateBlueprintVisitorHelper(searchable, field, requestContext), _field(field), _fieldId(fieldId), - _dictionary(dictionary) {} + _fieldIndexes(fieldIndexes) {} template <class TermNode> void visitTerm(TermNode &n) { const vespalib::string termStr = queryeval::termAsString(n); LOG(debug, "searching for '%s' in '%s'", termStr.c_str(), _field.getName().c_str()); - MemoryFieldIndex *fieldIndex = _dictionary.getFieldIndex(_fieldId); + FieldIndex *fieldIndex = _fieldIndexes.getFieldIndex(_fieldId); GenerationHandler::Guard genGuard = fieldIndex->takeGenerationGuard(); - Dictionary::PostingList::ConstIterator pitr - = fieldIndex->findFrozen(termStr); + FieldIndex::PostingList::ConstIterator pitr = fieldIndex->findFrozen(termStr); bool useBitVector = _field.isFilter(); - setResult(make_UP(new MemTermBlueprint(std::move(genGuard), pitr, - fieldIndex->getFeatureStore(), - _field, _fieldId, useBitVector))); + setResult(std::make_unique<MemTermBlueprint>(std::move(genGuard), pitr, + fieldIndex->getFeatureStore(), + _field, _fieldId, useBitVector)); } void visit(LocationTerm &n) override { visitTerm(n); } @@ -241,9 +237,9 @@ MemoryIndex::createBlueprint(const IRequestContext & requestContext, { uint32_t fieldId = _schema.getIndexFieldId(field.getName()); if (fieldId == Schema::UNKNOWN_FIELD_ID || _hiddenFields[fieldId]) { - return Blueprint::UP(new EmptyBlueprint(field)); + return std::make_unique<EmptyBlueprint>(field); } - CreateBlueprintVisitor visitor(*this, requestContext, field, fieldId, *_dictionary); + CreateBlueprintVisitor visitor(*this, requestContext, field, fieldId, *_fieldIndexes); const_cast<Node &>(term).accept(visitor); return visitor.getResult(); } @@ -252,28 +248,30 @@ MemoryUsage MemoryIndex::getMemoryUsage() const { MemoryUsage usage; - usage.merge(_dictionary->getMemoryUsage()); + usage.merge(_fieldIndexes->getMemoryUsage()); return usage; } uint64_t MemoryIndex::getNumWords() const { - return _dictionary->getNumUniqueWords(); + return _fieldIndexes->getNumUniqueWords(); } void MemoryIndex::pruneRemovedFields(const Schema &schema) { LockGuard lock(_lock); - if (_prunedSchema.get() == NULL) { - Schema::UP newSchema = Schema::intersect(_schema, schema); - if (_schema == *newSchema) + if (_prunedSchema.get() == nullptr) { + auto newSchema = Schema::intersect(_schema, schema); + if (_schema == *newSchema) { return; + } _prunedSchema.reset(newSchema.release()); } else { - Schema::UP newSchema = Schema::intersect(*_prunedSchema, schema); - if (*_prunedSchema == *newSchema) + auto newSchema = Schema::intersect(*_prunedSchema, schema); + if (*_prunedSchema == *newSchema) { return; + } _prunedSchema.reset(newSchema.release()); } SchemaUtil::IndexIterator i(_schema); diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index d12e844f35f..0b74e05c619 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -17,14 +17,13 @@ namespace document { class Document; } namespace search::memoryindex { class DocumentInverter; -class Dictionary; +class FieldIndexCollection; /** * Lock-free implementation of a memory-based index * using the document inverter and dictionary classes from searchlib. **/ -class MemoryIndex : public queryeval::Searchable -{ +class MemoryIndex : public queryeval::Searchable { private: index::Schema _schema; ISequencedTaskExecutor &_invertThreads; @@ -32,7 +31,7 @@ private: std::unique_ptr<DocumentInverter> _inverter0; std::unique_ptr<DocumentInverter> _inverter1; DocumentInverter *_inverter; - std::unique_ptr<Dictionary> _dictionary; + std::unique_ptr<FieldIndexCollection> _fieldIndexes; bool _frozen; uint32_t _maxDocId; uint32_t _numDocs; @@ -65,11 +64,8 @@ private: void flipInverter(); public: - /** - * Convenience type defs. - */ - typedef std::unique_ptr<MemoryIndex> UP; - typedef std::shared_ptr<MemoryIndex> SP; + using UP = std::unique_ptr<MemoryIndex>; + using SP = std::shared_ptr<MemoryIndex>; /** * Create a new memory index based on the given schema. @@ -139,15 +135,13 @@ public: void dump(index::IndexBuilder &indexBuilder); // implements Searchable - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpec &field, - const query::Node &term) override; - - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpecList &fields, - const query::Node &term) override { + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpec &field, + const query::Node &term) override; + + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpecList &fields, + const query::Node &term) override { return queryeval::Searchable::createBlueprint(requestContext, fields, term); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp index 1f15bcf1c75..9b127a8b096 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "ordereddocumentinserter.h" -#include "i_document_insert_listener.h" +#include "i_field_index_insert_listener.h" +#include "ordered_field_index_inserter.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -27,7 +27,7 @@ const vespalib::string emptyWord = ""; } -OrderedDocumentInserter::OrderedDocumentInserter(MemoryFieldIndex &fieldIndex) +OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex) : _word(), _prevDocId(noDocId), _prevAdd(false), @@ -39,14 +39,13 @@ OrderedDocumentInserter::OrderedDocumentInserter(MemoryFieldIndex &fieldIndex) { } -OrderedDocumentInserter::~OrderedDocumentInserter() +OrderedFieldIndexInserter::~OrderedFieldIndexInserter() { flush(); } - void -OrderedDocumentInserter::flushWord() +OrderedFieldIndexInserter::flushWord() { if (_removes.empty() && _adds.empty()) { return; @@ -68,17 +67,15 @@ OrderedDocumentInserter::flushWord() _adds.clear(); } - void -OrderedDocumentInserter::flush() +OrderedFieldIndexInserter::flush() { flushWord(); _listener.flush(); } - void -OrderedDocumentInserter::setNextWord(const vespalib::stringref word) +OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) { // TODO: Adjust here if zero length words should be legal. assert(_word < word); @@ -103,9 +100,8 @@ OrderedDocumentInserter::setNextWord(const vespalib::stringref word) assert(_word == wordStore.getWord(_dItr.getKey()._wordRef)); } - void -OrderedDocumentInserter::add(uint32_t docId, +OrderedFieldIndexInserter::add(uint32_t docId, const index::DocIdAndFeatures &features) { assert(docId != noDocId); @@ -118,9 +114,8 @@ OrderedDocumentInserter::add(uint32_t docId, _prevAdd = true; } - void -OrderedDocumentInserter::remove(uint32_t docId) +OrderedFieldIndexInserter::remove(uint32_t docId) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId); @@ -129,9 +124,8 @@ OrderedDocumentInserter::remove(uint32_t docId) _prevAdd = false; } - void -OrderedDocumentInserter::rewind() +OrderedFieldIndexInserter::rewind() { assert(_removes.empty() && _adds.empty()); _word = ""; @@ -140,9 +134,8 @@ OrderedDocumentInserter::rewind() _dItr.begin(); } - datastore::EntryRef -OrderedDocumentInserter::getWordRef() const +OrderedFieldIndexInserter::getWordRef() const { return _dItr.getKey()._wordRef; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h index 9645c3890e2..03cf3723f01 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h @@ -2,34 +2,36 @@ #pragma once -#include "iordereddocumentinserter.h" -#include "memoryfieldindex.h" +#include "i_ordered_field_index_inserter.h" +#include "field_index.h" #include <limits> namespace search::memoryindex { -class IDocumentInsertListener; - +class IFieldIndexInsertListener; /** - * Class for inserting updates to MemoryFieldIndex in an ordered manner - * (single pass scan of dictionary tree) + * Class used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. + * + * This is done by doing a single pass scan of the dictionary of the FieldIndex, + * and for each word updating the posting list with docId adds / removes. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { +private: vespalib::stringref _word; uint32_t _prevDocId; bool _prevAdd; - using DictionaryTree = MemoryFieldIndex::DictionaryTree; - using PostingListStore = MemoryFieldIndex::PostingListStore; - using KeyComp = MemoryFieldIndex::KeyComp; - using WordKey = MemoryFieldIndex::WordKey; - using PostingListKeyDataType = MemoryFieldIndex::PostingListKeyDataType; - MemoryFieldIndex &_fieldIndex; + using DictionaryTree = FieldIndex::DictionaryTree; + using PostingListStore = FieldIndex::PostingListStore; + using KeyComp = FieldIndex::KeyComp; + using WordKey = FieldIndex::WordKey; + using PostingListKeyDataType = FieldIndex::PostingListKeyDataType; + FieldIndex &_fieldIndex; DictionaryTree::Iterator _dItr; - IDocumentInsertListener &_listener; + IFieldIndexInsertListener &_listener; // Pending changes to posting list for (_word) std::vector<uint32_t> _removes; @@ -39,7 +41,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter static constexpr uint32_t noFieldId = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t noDocId = std::numeric_limits<uint32_t>::max(); - /* + /** * Flush pending changes to postinglist for (_word). * * _dItr is located at correct position. @@ -47,13 +49,13 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter void flushWord(); public: - OrderedDocumentInserter(MemoryFieldIndex &fieldIndex); - ~OrderedDocumentInserter() override; + OrderedFieldIndexInserter(FieldIndex &fieldIndex); + ~OrderedFieldIndexInserter() override; void setNextWord(const vespalib::stringref word) override; void add(uint32_t docId, const index::DocIdAndFeatures &features) override; void remove(uint32_t docId) override; - /* + /** * Flush pending changes to postinglist for (_word). Also flush * insert listener. * @@ -61,7 +63,7 @@ public: */ void flush() override; - /* + /** * Rewind iterator, to start new pass. */ void rewind() override; diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index 1f55aa466b1..4c29ec321e3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -1,26 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "postingiterator.h" +#include "posting_iterator.h" +#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.postingiterator"); +LOG_SETUP(".searchlib.memoryindex.posting_iterator"); -namespace search { -namespace memoryindex { +namespace search::memoryindex { -PostingIterator::PostingIterator(Dictionary::PostingList::ConstIterator itr, +PostingIterator::PostingIterator(FieldIndex::PostingList::ConstIterator itr, const FeatureStore & featureStore, uint32_t packedIndex, const fef::TermFieldMatchDataArray & matchData) : queryeval::RankedSearchIteratorBase(matchData), _itr(itr), _featureStore(featureStore), - _featureDecoder(NULL) + _featureDecoder(nullptr) { _featureStore.setupForField(packedIndex, _featureDecoder); } @@ -69,7 +68,5 @@ PostingIterator::doUnpack(uint32_t docId) setUnpacked(); } - -} // namespace search::memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h index 4960a3f299b..de337ef49f3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h @@ -2,19 +2,17 @@ #pragma once -#include "dictionary.h" +#include "field_index.h" #include <vespa/searchlib/queryeval/iterators.h> -namespace search { -namespace memoryindex { +namespace search::memoryindex { /** - * Search iterator for memory index posting list. - **/ -class PostingIterator : public queryeval::RankedSearchIteratorBase -{ + * Search iterator for memory field index posting list. + */ +class PostingIterator : public queryeval::RankedSearchIteratorBase { private: - Dictionary::PostingList::ConstIterator _itr; + FieldIndex::PostingList::ConstIterator _itr; const FeatureStore &_featureStore; FeatureStore::DecodeContextCooked _featureDecoder; @@ -27,7 +25,7 @@ public: * @param packedIndex the field or field collection owning features. * @param matchData the match data to unpack features into. **/ - PostingIterator(Dictionary::PostingList::ConstIterator itr, + PostingIterator(FieldIndex::PostingList::ConstIterator itr, const FeatureStore &featureStore, uint32_t packedIndex, const fef::TermFieldMatchDataArray &matchData); @@ -39,6 +37,5 @@ public: Trinary is_strict() const override { return Trinary::True; } }; -} // namespace search::memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index be216f7c2ba..c185ec93c9d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -1,24 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "urlfieldinverter.h" -#include "fieldinverter.h" +#include "field_inverter.h" +#include "url_field_inverter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> +#include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.urlfieldinverter"); +LOG_SETUP(".memoryindex.url_field_inverter"); -namespace search { - -namespace memoryindex { +namespace search::memoryindex { namespace { @@ -46,9 +44,7 @@ lowercaseToken(vespalib::string &dest, const char *src, size_t srcSize) return dest.size(); } - -} // namespace - +} using document::ArrayFieldValue; using document::DataType; @@ -64,7 +60,6 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; - void UrlFieldInverter::startDoc(uint32_t docId) { @@ -78,7 +73,6 @@ UrlFieldInverter::startDoc(uint32_t docId) _hostname->startDoc(docId); } - void UrlFieldInverter::endDoc() { @@ -92,7 +86,6 @@ UrlFieldInverter::endDoc() _hostname->endDoc(); } - void UrlFieldInverter::startElement(int32_t weight) { @@ -106,7 +99,6 @@ UrlFieldInverter::startElement(int32_t weight) _hostname->startElement(weight); } - void UrlFieldInverter::endElement() { @@ -120,7 +112,6 @@ UrlFieldInverter::endElement() _hostname->endElement(); } - void UrlFieldInverter::processUrlSubField(FieldInverter *inverter, const StructFieldValue &field, @@ -128,8 +119,9 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, bool addAnchors) { const FieldValue::UP sfv = field.getValue(subField); - if (!sfv) + if (!sfv) { return; + } if (!sfv->inherits(IDENTIFIABLE_CLASSID(StringFieldValue))) { LOG(error, "Illegal field type %s for URL subfield %s, expected string", @@ -137,7 +129,7 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, vespalib::string(subField).data()); return; } - const StringFieldValue &value = static_cast<const StringFieldValue &>(*sfv); + const auto &value = static_cast<const StringFieldValue &>(*sfv); if (addAnchors) { inverter->addWord(HOSTNAME_BEGIN); } @@ -147,7 +139,6 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, } } - void UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) { @@ -161,7 +152,6 @@ UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) processUrlSubField(_hostname, field, UrlDataType::FIELD_HOST, true); } - void UrlFieldInverter::processUrlField(const FieldValue &url_field) { @@ -172,11 +162,10 @@ UrlFieldInverter::processUrlField(const FieldValue &url_field) return; } assert(url_field.getClass().id() == StructFieldValue::classId); - const StructFieldValue &field = - static_cast<const StructFieldValue &>(url_field); + const auto &field = static_cast<const StructFieldValue &>(url_field); const FieldValue::UP all_val = field.getValue("all"); - if (all_val.get() == NULL) { + if (all_val.get() == nullptr) { if (_useAnnotations) { // New style, use annotations processAnnotatedUrlField(field); @@ -190,12 +179,11 @@ UrlFieldInverter::processUrlField(const FieldValue &url_field) all_val->getDataType()->getName().c_str()); return; } - const StringFieldValue &all_sfv = - static_cast<const StringFieldValue &>(*all_val); + const auto &all_sfv = static_cast<const StringFieldValue &>(*all_val); if (_useAnnotations) { StringFieldValue::SpanTrees trees = all_sfv.getSpanTrees(); const SpanTree *tree = StringFieldValue::findTree(trees, SPANTREE_NAME); - if (tree != NULL) { + if (tree != nullptr) { // New style, use annotations processAnnotatedUrlField(field); return; @@ -211,7 +199,9 @@ UrlFieldInverter::processUrlField(const FieldValue &url_field) processUrlOldStyle(s); } -void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { +void +UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) +{ URL url(reinterpret_cast<const unsigned char *>(s.data()), s.size()); _hostname->addWord(HOSTNAME_BEGIN); @@ -268,7 +258,6 @@ void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { _hostname->addWord(HOSTNAME_END); } - void UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) { @@ -280,7 +269,6 @@ UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) } } - void UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) { @@ -296,13 +284,16 @@ UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) } namespace { -bool isUriType(const DataType &type) { + +bool +isUriType(const DataType &type) +{ return type == UrlDataType::getInstance() - || type == *DataType::STRING - || type == *DataType::URI; + || type == *DataType::STRING + || type == *DataType::URI; } -} // namespace +} void UrlFieldInverter::invertUrlField(const FieldValue &val) @@ -320,7 +311,7 @@ UrlFieldInverter::invertUrlField(const FieldValue &val) break; case CollectionType::WEIGHTEDSET: if (cInfo.id() == WeightedSetFieldValue::classId) { - const WeightedSetFieldValue &wset = static_cast<const WeightedSetFieldValue &>(val); + const auto &wset = static_cast<const WeightedSetFieldValue &>(val); if (isUriType(wset.getNestedType())) { processWeightedSetUrlField(wset); } else { @@ -332,7 +323,7 @@ UrlFieldInverter::invertUrlField(const FieldValue &val) break; case CollectionType::ARRAY: if (cInfo.id() == ArrayFieldValue::classId) { - const ArrayFieldValue &arr = static_cast<const ArrayFieldValue&>(val); + const auto &arr = static_cast<const ArrayFieldValue&>(val); if (isUriType(arr.getNestedType())) { processArrayUrlField(arr); } else { @@ -370,7 +361,6 @@ UrlFieldInverter::removeDocument(uint32_t docId) _hostname->removeDocument(docId); } - UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, FieldInverter *all, FieldInverter *scheme, @@ -393,8 +383,5 @@ UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, { } - -} // namespace memoryindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h index 74f96cd40c3..1659e460af3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h @@ -5,16 +5,11 @@ #include <vespa/searchcommon/common/datatype.h> #include <vespa/document/fieldvalue/structfieldvalue.h> -namespace search -{ - -namespace memoryindex -{ +namespace search::memoryindex { class FieldInverter; -class UrlFieldInverter -{ +class UrlFieldInverter { FieldInverter *_all; FieldInverter *_scheme; FieldInverter *_host; @@ -35,11 +30,10 @@ class UrlFieldInverter void endElement(); - void - processUrlSubField(FieldInverter *inverter, - const document::StructFieldValue &field, - vespalib::stringref subField, - bool addAnchors); + void processUrlSubField(FieldInverter *inverter, + const document::StructFieldValue &field, + vespalib::stringref subField, + bool addAnchors); void processAnnotatedUrlField(const document::StructFieldValue &field); @@ -71,7 +65,4 @@ public: } }; - -} // namespace memoryindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index b65fe192e58..ffdc26f5eb0 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wordstore.h" +#include "word_store.h" #include <vespa/searchlib/datastore/datastore.hpp> namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h b/searchlib/src/vespa/searchlib/memoryindex/word_store.h index a9e941e04d1..4c1526df527 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.h @@ -7,11 +7,10 @@ namespace search::memoryindex { -class WordStore -{ +class WordStore { public: - typedef datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2> > DataStoreType; - typedef DataStoreType::RefType RefType; + using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; + using RefType = DataStoreType::RefType; private: DataStoreType _store; @@ -23,8 +22,7 @@ public: WordStore(); ~WordStore(); datastore::EntryRef addWord(const vespalib::stringref word); - const char * getWord(datastore::EntryRef ref) const - { + const char *getWord(datastore::EntryRef ref) const { RefType internalRef(ref); return _store.getEntry<char>(internalRef); } diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.h b/searchlib/src/vespa/searchlib/predicate/document_features_store.h index 2cf3e15337a..4c55b67cb11 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.h +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.h @@ -4,7 +4,7 @@ #include "predicate_tree_annotator.h" #include <vespa/searchlib/btree/btree.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/hash_map.h> #include <unordered_set> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 2d7a9abbf79..9cbbd136148 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -2,13 +2,13 @@ #include "fakememtreeocc.h" #include "fpfactory.h" -#include <vespa/searchlib/queryeval/iterators.h> -#include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> +#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/queryeval/iterators.h> #include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index ede14f237b2..f0363500559 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -4,8 +4,8 @@ #include "fakeword.h" #include "fakeposting.h" #include "fpfactory.h" -#include <vespa/searchlib/memoryindex/dictionary.h> -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> +#include <vespa/searchlib/memoryindex/field_index.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -15,7 +15,7 @@ namespace fakedata { class FakeMemTreeOccMgr : public FakeWord::RandomizedWriter { public: - typedef memoryindex::Dictionary::PostingList Tree; + typedef memoryindex::FieldIndex::PostingList Tree; typedef Tree::NodeAllocatorType NodeAllocator; typedef memoryindex::FeatureStore FeatureStore; typedef datastore::EntryRef EntryRef; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index 3c16fc8e9a8..33819d4f7cb 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -3,12 +3,13 @@ #include "fakezcfilterocc.h" #include "fpfactory.h" #include <vespa/searchlib/diskindex/zcposocciterators.h> -#include <vespa/searchlib/diskindex/zcbuf.h> +#include <vespa/searchlib/diskindex/zc4_posting_writer.h> using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataArray; using search::fef::TermFieldMatchDataPosition; using search::queryeval::SearchIterator; +using search::index::PostingListCounts; using search::index::PostingListParams; using search::index::DocIdAndFeatures; using search::index::DocIdAndPosOccFeatures; @@ -24,11 +25,6 @@ namespace search { namespace fakedata { -#define L1SKIPSTRIDE 16 -#define L2SKIPSTRIDE 8 -#define L3SKIPSTRIDE 8 -#define L4SKIPSTRIDE 8 - #define DEBUG_ZCFILTEROCC_PRINTF 0 #define DEBUG_ZCFILTEROCC_ASSERT 0 @@ -137,35 +133,8 @@ void FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, bool dynamicK) { - ZcBuf bytes; - ZcBuf l1SkipBytes; - ZcBuf l2SkipBytes; - ZcBuf l3SkipBytes; - ZcBuf l4SkipBytes; - uint32_t lastDocId = 0u; - uint32_t lastL1SkipDocId = 0u; - uint64_t lastL1SkipDocIdPos = 0; - uint64_t lastL1SkipFeaturePos = 0; - unsigned int l1SkipCnt = 0; - uint32_t lastL2SkipDocId = 0u; - uint64_t lastL2SkipDocIdPos = 0; - uint64_t lastL2SkipFeaturePos = 0; - uint64_t lastL2SkipL1SkipPos = 0; - unsigned int l2SkipCnt = 0; - uint32_t lastL3SkipDocId = 0u; - uint64_t lastL3SkipDocIdPos = 0; - uint64_t lastL3SkipFeaturePos = 0; - uint64_t lastL3SkipL1SkipPos = 0; - uint64_t lastL3SkipL2SkipPos = 0; - unsigned int l3SkipCnt = 0; - uint32_t lastL4SkipDocId = 0u; - uint64_t lastL4SkipDocIdPos = 0; - uint64_t lastL4SkipFeaturePos = 0; - uint64_t lastL4SkipL1SkipPos = 0; - uint64_t lastL4SkipL2SkipPos = 0; - uint64_t lastL4SkipL3SkipPos = 0; - unsigned int l4SkipCnt = 0; - uint64_t featurePos = 0; + PostingListCounts counts; + Zc4PostingWriter<bigEndian> writer(counts); typedef FakeWord FW; typedef FW::DocWordFeatureList DWFL; @@ -181,288 +150,88 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, FeatureEncodeContext<bigEndian> &f = (dynamicK ? static_cast<FeatureEncodeContext<bigEndian> &>(f1) : static_cast<FeatureEncodeContext<bigEndian> &>(f0)); - search::ComprFileWriteContext fctx(f); - f.setWriteContext(&fctx); - fctx.allocComprBuf(64, 1); - f.afterWrite(fctx, 0, 0); + writer.set_dynamic_k(dynamicK); + if (doFeatures) { + writer.set_encode_features(&f); + } + PostingListParams params; + params.set("docIdLimit", fw._docIdLimit); + params.set("minChunkDocs", 1000000000); // Disable chunking + params.set("minSkipDocs", 1u); // Force skip info + writer.set_posting_list_params(params); + auto &writeContext = writer.get_write_context(); + search::ComprBuffer &cb = writeContext; + auto &e = writer.get_encode_context(); + writeContext.allocComprBuf(65536u, 32768u); + e.setupWrite(cb); // Ensure that some space is initially available in encoding buffers - bytes.maybeExpand(); - l1SkipBytes.maybeExpand(); - l2SkipBytes.maybeExpand(); - l3SkipBytes.maybeExpand(); - l4SkipBytes.maybeExpand(); while (d != de) { - if (l1SkipCnt >= L1SKIPSTRIDE) { - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l1SkipBytes.encode(docIdDelta - 1); - uint64_t lastDocIdPos = bytes.size(); - uint32_t docIdPosDelta = lastDocIdPos - lastL1SkipDocIdPos; - l1SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - featurePos = f.getWriteOffset(); - l1SkipBytes.encode(featurePos - lastL1SkipFeaturePos - 1); - lastL1SkipFeaturePos = featurePos; - } -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L1Encode docId=%d (+%d), docIdPos=%d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta); -#endif - lastL1SkipDocId = lastDocId; - lastL1SkipDocIdPos = lastDocIdPos; - l1SkipCnt = 0; - ++l2SkipCnt; - if (l2SkipCnt >= L2SKIPSTRIDE) { - docIdDelta = lastDocId - lastL2SkipDocId; - docIdPosDelta = lastDocIdPos - lastL2SkipDocIdPos; - uint64_t lastL1SkipPos = l1SkipBytes.size(); - uint32_t l1SkipPosDelta = lastL1SkipPos - lastL2SkipL1SkipPos; - l2SkipBytes.encode(docIdDelta - 1); - l2SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - l2SkipBytes.encode(featurePos - lastL2SkipFeaturePos - 1); - lastL2SkipFeaturePos = featurePos; - } - l2SkipBytes.encode(l1SkipPosDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L2Encode docId=%d (+%d), docIdPos=%d (+%u)," - " l1SkipPos=%d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta, - (int) lastL1SkipPos, l1SkipPosDelta); -#endif - lastL2SkipDocId = lastDocId; - lastL2SkipDocIdPos = lastDocIdPos; - lastL2SkipL1SkipPos = lastL1SkipPos; - l2SkipCnt = 0; - ++l3SkipCnt; - if (l3SkipCnt >= L3SKIPSTRIDE) { - docIdDelta = lastDocId - lastL3SkipDocId; - docIdPosDelta = lastDocIdPos - lastL3SkipDocIdPos; - l1SkipPosDelta = lastL1SkipPos - lastL3SkipL1SkipPos; - uint64_t lastL2SkipPos = l2SkipBytes.size(); - uint32_t l2SkipPosDelta = lastL2SkipPos - - lastL3SkipL2SkipPos; - l3SkipBytes.encode(docIdDelta - 1); - l3SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - l3SkipBytes.encode(featurePos - lastL3SkipFeaturePos - 1); - lastL3SkipFeaturePos = featurePos; - } - l3SkipBytes.encode(l1SkipPosDelta - 1); - l3SkipBytes.encode(l2SkipPosDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L3Encode docId=%d (+%d), docIdPos=%d (+%u)," - " l1SkipPos=%d (+%u) l2SkipPos %d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta, - (int) lastL1SkipPos, l1SkipPosDelta, - (int) lastL2SkipPos, l2SkipPosDelta); -#endif - lastL3SkipDocId = lastDocId; - lastL3SkipDocIdPos = lastDocIdPos; - lastL3SkipL1SkipPos = lastL1SkipPos; - lastL3SkipL2SkipPos = lastL2SkipPos; - l3SkipCnt = 0; - ++l4SkipCnt; - if (l4SkipCnt >= L4SKIPSTRIDE) { - docIdDelta = lastDocId - lastL4SkipDocId; - docIdPosDelta = lastDocIdPos - lastL4SkipDocIdPos; - l1SkipPosDelta = lastL1SkipPos - lastL4SkipL1SkipPos; - l2SkipPosDelta = lastL2SkipPos - lastL4SkipL2SkipPos; - uint64_t lastL3SkipPos = l3SkipBytes.size(); - uint32_t l3SkipPosDelta = lastL3SkipPos - - lastL4SkipL3SkipPos; - l4SkipBytes.encode(docIdDelta - 1); - l4SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - l4SkipBytes.encode(featurePos - lastL4SkipFeaturePos - 1); - lastL4SkipFeaturePos = featurePos; - } - l4SkipBytes.encode(l1SkipPosDelta - 1); - l4SkipBytes.encode(l2SkipPosDelta - 1); - l4SkipBytes.encode(l3SkipPosDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L4Encode docId=%d (+%d), docIdPos=%d (+%u)," - " l1SkipPos=%d (+%u) l2SkipPos %d (+%u)" - " l3SkipPos=%d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta, - (int) lastL1SkipPos, l1SkipPosDelta, - (int) lastL2SkipPos, l2SkipPosDelta, - (int) lastL3SkipPos, l3SkipPosDelta); -#endif - lastL4SkipDocId = lastDocId; - lastL4SkipDocIdPos = lastDocIdPos; - lastL4SkipL1SkipPos = lastL1SkipPos; - lastL4SkipL2SkipPos = lastL2SkipPos; - lastL4SkipL3SkipPos = lastL3SkipPos; - l4SkipCnt = 0; - } - } - } - } - if (lastDocId == 0u) { - bytes.encode(d->_docId - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("Encode docId=%d\n", - d->_docId); -#endif - } else { - uint32_t docIdDelta = d->_docId - lastDocId; - bytes.encode(docIdDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("Encode docId=%d (+%d)\n", - d->_docId, docIdDelta); -#endif - } if (doFeatures) { fw.setupFeatures(*d, &*p, features); p += d->_positions; - f.writeFeatures(features); + } else { + features.clear(d->_docId); } - lastDocId = d->_docId; - ++l1SkipCnt; + writer.write_docid_and_features(features); ++d; } if (doFeatures) { assert(p == pe); - _featuresSize = f.getWriteOffset(); - // First pad to 64 bits. - uint32_t pad = (64 - f.getWriteOffset()) & 63; - while (pad > 0) { - uint32_t now = std::min(32u, pad); - f.writeBits(0, now); - f.writeComprBufferIfNeeded(); - pad -= now; - } - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - for (unsigned int i = 0; i < 4; i++) { - f.writeBits(0, 32); - f.writeComprBufferIfNeeded(); - } - f.writeComprBufferIfNeeded(); - f.flush(); - f.writeComprBuffer(); - } else { - _featuresSize = 0; - } - // Extra partial entries for skip tables to simplify iterator during search - if (l1SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l1SkipBytes.encode(docIdDelta - 1); - } - if (l2SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL2SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l2SkipBytes.encode(docIdDelta - 1); - } - if (l3SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL3SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l3SkipBytes.encode(docIdDelta - 1); - } - if (l4SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL4SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l4SkipBytes.encode(docIdDelta - 1); } + writer.flush_word(); + _featuresSize = 0; _hitDocs = fw._postings.size(); _docIdLimit = fw._docIdLimit; - _lastDocId = lastDocId; - FeatureEncodeContext<bigEndian> e; - ComprFileWriteContext ectx(e); - e.setWriteContext(&ectx); - ectx.allocComprBuf(64, 1); - e.afterWrite(ectx, 0, 0); + _compressedBits = e.getWriteOffset(); + assert(_compressedBits == counts._bitLength); + assert(_hitDocs == counts._numDocs); + _lastDocId = fw._postings.back()._docId; + writer.on_close(); - // Encode word header - e.encodeExpGolomb(_hitDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - _docIdsSize = bytes.size() * 8; - _l1SkipSize = l1SkipBytes.size(); - _l2SkipSize = _l3SkipSize = _l4SkipSize = 0; - if (_l1SkipSize != 0) - _l2SkipSize = l2SkipBytes.size(); - if (_l2SkipSize != 0) - _l3SkipSize = l3SkipBytes.size(); - if (_l3SkipSize != 0) - _l4SkipSize = l4SkipBytes.size(); - - e.encodeExpGolomb(bytes.size() - 1, K_VALUE_ZCPOSTING_DOCIDSSIZE); - e.encodeExpGolomb(_l1SkipSize, K_VALUE_ZCPOSTING_L1SKIPSIZE); - e.writeComprBufferIfNeeded(); + std::pair<void *, size_t> ectxData = writeContext.grabComprBuffer(_compressedMalloc); + _compressed = std::make_pair(static_cast<uint64_t *>(ectxData.first), + ectxData.second); + read_header<bigEndian>(doFeatures, dynamicK, writer.get_min_skip_docs(), writer.get_min_chunk_docs()); +} + +template <bool bigEndian> +void +FakeZcFilterOcc::read_header(bool doFeatures, bool dynamicK, uint32_t min_skip_docs, uint32_t min_chunk_docs) +{ + // read back word header to get skip sizes + using EC = FeatureEncodeContext<bigEndian>; + UC64_DECODECONTEXT(o); + uint32_t length; + uint64_t val64; + UC64_SETUPBITS_NS(o, _compressed.first, 0, EC); + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_NUMDOCS, EC); + assert(static_cast<uint32_t>(val64) + 1 == _hitDocs); + assert(_hitDocs >= min_skip_docs); + assert(_hitDocs < min_chunk_docs); + uint32_t docIdK = dynamicK ? EC::calcDocIdK(_hitDocs, _docIdLimit) : K_VALUE_ZCPOSTING_LASTDOCID; + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_DOCIDSSIZE, EC); + _docIdsSize = val64 + 1; + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L1SKIPSIZE, EC); + _l1SkipSize = val64; if (_l1SkipSize != 0) { - e.encodeExpGolomb(_l2SkipSize, K_VALUE_ZCPOSTING_L2SKIPSIZE); - if (_l2SkipSize != 0) { - e.writeComprBufferIfNeeded(); - e.encodeExpGolomb(_l3SkipSize, K_VALUE_ZCPOSTING_L3SKIPSIZE); - if (_l3SkipSize != 0) { - e.encodeExpGolomb(_l4SkipSize, K_VALUE_ZCPOSTING_L4SKIPSIZE); - } - } + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L2SKIPSIZE, EC); + _l2SkipSize = val64; } - e.writeComprBufferIfNeeded(); - if (doFeatures) { - e.encodeExpGolomb(_featuresSize, K_VALUE_ZCPOSTING_FEATURESSIZE); - } - uint32_t docIdK = e.calcDocIdK(_hitDocs, _docIdLimit); - if (dynamicK) - e.encodeExpGolomb(_docIdLimit - 1 - _lastDocId, docIdK); - else - e.encodeExpGolomb(_docIdLimit - 1 - _lastDocId, - K_VALUE_ZCPOSTING_LASTDOCID); - uint64_t bytePad = (- e.getWriteOffset()) & 7; - if (bytePad > 0) - e.writeBits(0, bytePad); - size_t docIdSize = bytes.size(); - if (docIdSize > 0) { - writeZcBuf(e, bytes); + if (_l2SkipSize != 0) { + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L3SKIPSIZE, EC); + _l3SkipSize = val64; } - if (_l1SkipSize > 0) { - writeZcBuf(e, l1SkipBytes); - if (_l2SkipSize > 0) { - writeZcBuf(e, l2SkipBytes); - if (_l3SkipSize > 0) { - writeZcBuf(e, l3SkipBytes); - if (_l4SkipSize > 0) { - writeZcBuf(e, l4SkipBytes); - } - } - } + if (_l3SkipSize != 0) { + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L4SKIPSIZE, EC); + _l4SkipSize = val64; } if (doFeatures) { - e.writeBits(static_cast<const uint64_t *>(fctx._comprBuf), - 0, - _featuresSize); + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_FEATURESSIZE, EC); + _featuresSize = val64; } - _compressedBits = e.getWriteOffset(); - // First pad to 64 bits. - uint32_t pad = (64 - e.getWriteOffset()) & 63; - while (pad > 0) { - uint32_t now = std::min(32u, pad); - e.writeBits(0, now); - e.writeComprBufferIfNeeded(); - pad -= now; - } - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - for (unsigned int i = 0; i < 4; i++) { - e.writeBits(0, 32); - e.writeComprBufferIfNeeded(); - } - e.writeComprBufferIfNeeded(); - e.flush(); - e.writeComprBuffer(); - - std::pair<void *, size_t> ectxData = ectx.grabComprBuffer(_compressedMalloc); - _compressed = std::make_pair(static_cast<uint64_t *>(ectxData.first), - ectxData.second); + UC64_DECODEEXPGOLOMB_NS(o, docIdK, EC); + assert(_lastDocId == _docIdLimit - 1 - val64); } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h index d5df198acdc..b68e3866461 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h @@ -37,6 +37,9 @@ protected: template <bool bigEndian> void setupT(const FakeWord &fw, bool doFeatures, bool dynamicK); + template <bool bigEndian> + void read_header(bool do_features, bool dynamic_k, uint32_t min_skip_docs, uint32_t min_cunk_docs); + public: FakeZcFilterOcc(const FakeWord &fw); FakeZcFilterOcc(const FakeWord &fw, bool bigEndian, const char *nameSuffix); diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h index 4802a7571c2..08473f9fc6c 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h @@ -2,20 +2,12 @@ #pragma once -#include <vespa/searchlib/memoryindex/iordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h> #include <sstream> -namespace search -{ +namespace search::memoryindex::test { -namespace memoryindex -{ - -namespace test -{ - -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { std::stringstream _ss; bool _first; bool _verbose; @@ -31,7 +23,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter } } public: - OrderedDocumentInserter() + OrderedFieldIndexInserter() : _ss(), _first(true), _verbose(false), @@ -115,6 +107,4 @@ public: void setVerbose() { _verbose = true; } }; -} // namespace test -} // namespace memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 26c66f43993..155bb194f97 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -409,88 +409,6 @@ ComprFileReadContext::referenceWriteContext(const ComprFileWriteContext &rhs) } -void -ComprFileReadContext::copyWriteContext(const ComprFileWriteContext &rhs) -{ - ComprFileEncodeContext *e = rhs.getEncodeContext(); - ComprFileDecodeContext *d = getDecodeContext(); - - assert(e != NULL); - int usedUnits = e->getUsedUnits(rhs._comprBuf); - assert(usedUnits >= 0); - - dropComprBuf(); - allocComprBuf(usedUnits, 32768); - assert(_comprBufSize >= static_cast<unsigned int>(usedUnits)); - memcpy(_comprBuf, rhs._comprBuf, - static_cast<size_t>(usedUnits) * _unitSize); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - - -void -ComprFileReadContext::referenceReadContext(const ComprFileReadContext &rhs) -{ - ComprFileDecodeContext *d = getDecodeContext(); - - int usedUnits = rhs.getBufferEndFilePos() / _unitSize; - assert(usedUnits >= 0); - assert(static_cast<uint64_t>(usedUnits) * _unitSize == - rhs.getBufferEndFilePos()); - - referenceComprBuf(rhs); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - - -void -ComprFileReadContext::copyReadContext(const ComprFileReadContext &rhs) -{ - ComprFileDecodeContext *d = getDecodeContext(); - - int usedUnits = rhs.getBufferEndFilePos() / _unitSize; - assert(usedUnits >= 0); - assert(static_cast<uint64_t>(usedUnits) * _unitSize == - rhs.getBufferEndFilePos()); - - dropComprBuf(); - allocComprBuf(usedUnits, 32768); - assert(_comprBufSize >= static_cast<unsigned int>(usedUnits)); - memcpy(_comprBuf, rhs._comprBuf, - static_cast<size_t>(usedUnits) * _unitSize); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - ComprFileWriteContext:: ComprFileWriteContext(ComprFileEncodeContext &encodeContext) : ComprBuffer(encodeContext.getUnitByteSize()), diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index 3d44f088c74..d4de1d305fa 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -136,23 +136,7 @@ public: * For unit testing only. Reference data owned by rhs, only works as * long as rhs is live and unchanged. */ - void referenceReadContext(const ComprFileReadContext &rhs); - - /* - * For unit testing only. Copy data owned by rhs. - */ - void copyReadContext(const ComprFileReadContext &rhs); - - /* - * For unit testing only. Reference data owned by rhs, only works as - * long as rhs is live and unchanged. - */ void referenceWriteContext(const ComprFileWriteContext &rhs); - - /* - * For unit testing only. Copy data owned by rhs. - */ - void copyWriteContext(const ComprFileWriteContext &rhs); }; diff --git a/security-utils/pom.xml b/security-utils/pom.xml index f7704762250..10dec598915 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -31,16 +31,6 @@ <artifactId>jackson-databind</artifactId> <scope>compile</scope> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <scope>compile</scope> - </dependency> <!-- test scope --> <dependency> diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java index 2ea1e1efe83..f5f9182fc4e 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java @@ -1,9 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; -import com.yahoo.security.tls.https.TlsAwareHttpClientBuilder; - -import java.net.http.HttpClient; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; @@ -51,12 +48,6 @@ public class TransportSecurityUtils { .map(configFile -> new ReloadingTlsContext(configFile, getInsecureAuthorizationMode())); } - public static HttpClient.Builder createHttpClientBuilder(String userAgent) { - return createTlsContext() - .map(tlsContext -> new TlsAwareHttpClientBuilder(tlsContext, userAgent)) - .orElseGet(() -> new TlsAwareHttpClientBuilder(userAgent)); - } - private static Optional<String> getEnvironmentVariable(String environmentVariable) { return Optional.ofNullable(System.getenv(environmentVariable)) .filter(var -> !var.isEmpty()); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpClient.java b/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpClient.java deleted file mode 100644 index 2911b77707a..00000000000 --- a/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpClient.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.security.tls.https; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLParameters; -import java.io.IOException; -import java.net.Authenticator; -import java.net.CookieHandler; -import java.net.ProxySelector; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; - -/** - * A {@link HttpClient} that uses either http or https based on the global Vespa TLS configuration. - * - * @author bjorncs - */ -class TlsAwareHttpClient extends HttpClient { - - private final HttpClient wrappedClient; - private final String userAgent; - - TlsAwareHttpClient(HttpClient wrappedClient, String userAgent) { - this.wrappedClient = wrappedClient; - this.userAgent = userAgent; - } - - @Override - public Optional<CookieHandler> cookieHandler() { - return wrappedClient.cookieHandler(); - } - - @Override - public Optional<Duration> connectTimeout() { - return wrappedClient.connectTimeout(); - } - - @Override - public Redirect followRedirects() { - return wrappedClient.followRedirects(); - } - - @Override - public Optional<ProxySelector> proxy() { - return wrappedClient.proxy(); - } - - @Override - public SSLContext sslContext() { - return wrappedClient.sslContext(); - } - - @Override - public SSLParameters sslParameters() { - return wrappedClient.sslParameters(); - } - - @Override - public Optional<Authenticator> authenticator() { - return wrappedClient.authenticator(); - } - - @Override - public Version version() { - return wrappedClient.version(); - } - - @Override - public Optional<Executor> executor() { - return wrappedClient.executor(); - } - - @Override - public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> responseBodyHandler) throws IOException, InterruptedException { - return wrappedClient.send(wrapRequest(request), responseBodyHandler); - } - - @Override - public <T> CompletableFuture<HttpResponse<T>> sendAsync(HttpRequest request, HttpResponse.BodyHandler<T> responseBodyHandler) { - return wrappedClient.sendAsync(wrapRequest(request), responseBodyHandler); - } - - @Override - public <T> CompletableFuture<HttpResponse<T>> sendAsync(HttpRequest request, HttpResponse.BodyHandler<T> responseBodyHandler, HttpResponse.PushPromiseHandler<T> pushPromiseHandler) { - return wrappedClient.sendAsync(wrapRequest(request), responseBodyHandler, pushPromiseHandler); - } - - @Override - public String toString() { - return wrappedClient.toString(); - } - - private HttpRequest wrapRequest(HttpRequest request) { - return new TlsAwareHttpRequest(request, userAgent); - } -} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpClientBuilder.java b/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpClientBuilder.java deleted file mode 100644 index 5a375cf663f..00000000000 --- a/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpClientBuilder.java +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.security.tls.https; - -import com.yahoo.security.tls.TlsContext; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLParameters; -import java.net.Authenticator; -import java.net.CookieHandler; -import java.net.ProxySelector; -import java.net.http.HttpClient; -import java.time.Duration; -import java.util.concurrent.Executor; - -/** - * A client builder for {@link HttpClient} which uses {@link TlsContext} for TLS configuration. - * Intended for internal Vespa communication only. - * - * @author bjorncs - */ -public class TlsAwareHttpClientBuilder implements HttpClient.Builder { - - private final HttpClient.Builder wrappedBuilder; - private final String userAgent; - - public TlsAwareHttpClientBuilder(String userAgent) { - this(null, userAgent); - } - - public TlsAwareHttpClientBuilder(TlsContext tlsContext, String userAgent) { - this.wrappedBuilder = tlsContext != null ? - HttpClient.newBuilder().sslContext(tlsContext.context()).sslParameters(tlsContext.parameters()) : - HttpClient.newBuilder(); - this.userAgent = userAgent; - } - - @Override - public HttpClient.Builder cookieHandler(CookieHandler cookieHandler) { - throw new UnsupportedOperationException(); - } - - @Override - public HttpClient.Builder connectTimeout(Duration duration) { - wrappedBuilder.connectTimeout(duration); - return this; - } - - @Override - public HttpClient.Builder sslContext(SSLContext sslContext) { - throw new UnsupportedOperationException("SSLContext is given from tls context"); - } - - @Override - public HttpClient.Builder sslParameters(SSLParameters sslParameters) { - throw new UnsupportedOperationException("SSLParameters is given from tls context"); - } - - @Override - public HttpClient.Builder executor(Executor executor) { - wrappedBuilder.executor(executor); - return this; - } - - @Override - public HttpClient.Builder followRedirects(HttpClient.Redirect policy) { - wrappedBuilder.followRedirects(policy); - return this; - } - - @Override - public HttpClient.Builder version(HttpClient.Version version) { - wrappedBuilder.version(version); - return this; - } - - @Override - public HttpClient.Builder priority(int priority) { - wrappedBuilder.priority(priority); - return this; - } - - @Override - public HttpClient.Builder proxy(ProxySelector proxySelector) { - throw new UnsupportedOperationException(); - } - - @Override - public HttpClient.Builder authenticator(Authenticator authenticator) { - throw new UnsupportedOperationException(); - } - - @Override - public HttpClient build() { - // TODO Stop wrapping the client once TLS is mandatory - return new TlsAwareHttpClient(wrappedBuilder.build(), userAgent); - } -} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpRequest.java b/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpRequest.java deleted file mode 100644 index bbdd8af791f..00000000000 --- a/security-utils/src/main/java/com/yahoo/security/tls/https/TlsAwareHttpRequest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.security.tls.https; - -import com.yahoo.security.tls.MixedMode; -import com.yahoo.security.tls.TransportSecurityUtils; - -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpHeaders; -import java.net.http.HttpRequest; -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; - -/** - * A {@link HttpRequest} where the scheme is either http or https based on the global Vespa TLS configuration. - * - * @author bjorncs - */ -class TlsAwareHttpRequest extends HttpRequest { - - private final URI rewrittenUri; - private final HttpRequest wrappedRequest; - private final HttpHeaders rewrittenHeaders; - - TlsAwareHttpRequest(HttpRequest wrappedRequest, String userAgent) { - this.wrappedRequest = wrappedRequest; - this.rewrittenUri = rewriteUri(wrappedRequest.uri()); - this.rewrittenHeaders = rewriteHeaders(wrappedRequest, userAgent); - } - - @Override - public Optional<BodyPublisher> bodyPublisher() { - return wrappedRequest.bodyPublisher(); - } - - @Override - public String method() { - return wrappedRequest.method(); - } - - @Override - public Optional<Duration> timeout() { - return wrappedRequest.timeout(); - } - - @Override - public boolean expectContinue() { - return wrappedRequest.expectContinue(); - } - - @Override - public URI uri() { - return rewrittenUri; - } - - @Override - public Optional<HttpClient.Version> version() { - return wrappedRequest.version(); - } - - @Override - public HttpHeaders headers() { - return rewrittenHeaders; - } - - private static URI rewriteUri(URI uri) { - if (!uri.getScheme().equals("http")) { - return uri; - } - String rewrittenScheme = - TransportSecurityUtils.getConfigFile().isPresent() && TransportSecurityUtils.getInsecureMixedMode() != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER ? - "https" : - "http"; - int port = uri.getPort(); - int rewrittenPort = port != -1 ? port : (rewrittenScheme.equals("http") ? 80 : 443); - try { - return new URI(rewrittenScheme, uri.getUserInfo(), uri.getHost(), rewrittenPort, uri.getPath(), uri.getQuery(), uri.getFragment()); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - - private static HttpHeaders rewriteHeaders(HttpRequest request, String userAgent) { - HttpHeaders headers = request.headers(); - if (headers.firstValue("User-Agent").isPresent()) { - return headers; - } - HashMap<String, List<String>> rewrittenHeaders = new HashMap<>(headers.map()); - rewrittenHeaders.put("User-Agent", List.of(userAgent)); - return HttpHeaders.of(rewrittenHeaders, (ignored1, ignored2) -> true); - } - - @Override - public String toString() { - return "TlsAwareHttpRequest{" + - "rewrittenUri=" + rewrittenUri + - ", wrappedRequest=" + wrappedRequest + - '}'; - } -} diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 0826826f433..578fcc83006 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -16,6 +16,22 @@ <description>Service monitor component for hosted vespa.</description> <dependencies> + <!-- compile scope --> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>4.5</version> + <!-- This is necessary to get 4.4's HostnameVerifier API of SSLConnectionSocketFactory::new --> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>http-utils</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + + <!-- provided scope --> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config</artifactId> @@ -68,6 +84,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -97,13 +114,8 @@ <artifactId>jackson-databind</artifactId> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>4.5</version> - <!-- This is necessary to get 4.4's HostnameVerifier API of SSLConnectionSocketFactory::new --> - <scope>compile</scope> - </dependency> + + <!-- test scope --> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java index 4a382ee8d94..59144b15539 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApacheHttpClient.java @@ -1,19 +1,14 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.util.http.VespaHttpClientBuilder; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.BasicHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import java.io.IOException; @@ -32,13 +27,7 @@ class ApacheHttpClient implements AutoCloseable { T handle(CloseableHttpResponse httpResponse) throws Exception; } - static CloseableHttpClient makeCloseableHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) { - Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() - .register(url.getProtocol(),socketFactory) - .build(); - - HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(registry); - + static CloseableHttpClient makeCloseableHttpClient(Duration timeout, Duration keepAlive) { RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout((int) timeout.toMillis()) // establishment of connection .setConnectionRequestTimeout((int) timeout.toMillis()) // connection from connection manager @@ -57,16 +46,15 @@ class ApacheHttpClient implements AutoCloseable { } }; - return HttpClients.custom() + return VespaHttpClientBuilder.createWithBasicConnectionManager() .setKeepAliveStrategy(keepAliveStrategy) - .setConnectionManager(connectionManager) .disableAutomaticRetries() .setDefaultRequestConfig(requestConfig) .build(); } - ApacheHttpClient(URL url, Duration timeout, Duration keepAlive, ConnectionSocketFactory socketFactory) { - this(url, makeCloseableHttpClient(url, timeout, keepAlive, socketFactory)); + ApacheHttpClient(URL url, Duration timeout, Duration keepAlive) { + this(url, makeCloseableHttpClient(timeout, keepAlive)); } ApacheHttpClient(URL url, CloseableHttpClient client) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java index ede11c48de4..19399c79b43 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.log.LogLevel; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.util.EntityUtils; import java.io.IOException; @@ -30,7 +29,7 @@ public class StateV1HealthClient implements AutoCloseable { private final Function<HttpEntity, String> getContentFunction; StateV1HealthClient(URL url, Duration requestTimeout, Duration connectionKeepAlive) { - this(new ApacheHttpClient(url, requestTimeout, connectionKeepAlive, PlainConnectionSocketFactory.getSocketFactory()), + this(new ApacheHttpClient(url, requestTimeout, connectionKeepAlive), entity -> uncheck(() -> EntityUtils.toString(entity))); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index d81c9f064b1..da3bd18440b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; @@ -55,7 +55,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); HttpUriRequest request = RequestBuilder.put() .setUri(uri) @@ -66,7 +66,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); HttpUriRequest request = RequestBuilder.delete() .setUri(uri) @@ -76,7 +76,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); HttpUriRequest request = RequestBuilder.put() .setUri(uri) @@ -87,7 +87,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); HttpUriRequest request = RequestBuilder.delete() .setUri(uri) diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index cf044edeac0..e78478bc1a2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -17,13 +17,13 @@ import java.util.Set; */ public interface ZmsClient extends AutoCloseable { - void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token); - void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token); - void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); + void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); - void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token); + void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token); boolean getMembership(AthenzRole role, AthenzIdentity identity); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java index dccd18fed61..a67bd4dcad6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java @@ -33,7 +33,7 @@ public class ProviderResourceGroupRolesRequestEntity { @JsonProperty("resourceGroup") private final String resourceGroup; - public ProviderResourceGroupRolesRequestEntity(AthenzService providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { + public ProviderResourceGroupRolesRequestEntity(AthenzIdentity providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { this.domain = providerService.getDomainName(); this.service = providerService.getName(); this.tenant = tenantDomain.getName(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java index 7883a505c71..6e1987130f2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.client.zms.bindings; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import java.util.List; @@ -23,7 +24,7 @@ public class TenancyRequestEntity { @JsonInclude(JsonInclude.Include.NON_EMPTY) private final List<String> resourceGroups; - public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzService providerService, List<String> resourceGroups) { + public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzIdentity providerService, List<String> resourceGroups) { this.tenantDomain = tenantDomain.getName(); this.providerService = providerService.getFullName(); this.resourceGroups = resourceGroups; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 05395947fc1..ddba229d8d1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -1,10 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.NToken; @@ -22,7 +22,6 @@ import com.yahoo.vespa.athenz.client.zts.bindings.RoleTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.TenantDomainsResponseEntity; import com.yahoo.vespa.athenz.client.zts.utils.IdentityCsrGenerator; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.security.Pkcs10Csr; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -65,8 +64,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public InstanceIdentity registerInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceIdentity registerInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, @@ -81,8 +80,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public InstanceIdentity refreshInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { @@ -101,7 +100,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr) { URI uri = ztsUrl.resolve(String.format("instance/%s/%s/refresh", identity.getDomainName(), identity.getName())); HttpUriRequest request = RequestBuilder.post() .setUri(uri) @@ -114,7 +113,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix) { Pkcs10Csr csr = new IdentityCsrGenerator(dnsSuffix).generateIdentityCsr(identity, keyPair); return getServiceIdentity(identity, keyId, csr); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index 7b77fccfed6..efe244d500f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -28,8 +28,8 @@ public interface ZtsClient extends AutoCloseable { * @param attestationData The signed identity documented serialized to a string. * @return A x509 certificate + service token (optional) */ - InstanceIdentity registerInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + InstanceIdentity registerInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, // TODO Remove this parameter (unused/unnecessary) String attestationData, boolean requestServiceToken, @@ -40,8 +40,8 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate + service token (optional) */ - InstanceIdentity refreshInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr); @@ -51,7 +51,7 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate with CA certificates */ - Identity getServiceIdentity(AthenzService identity, + Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr); @@ -60,7 +60,7 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate with CA certificates */ - Identity getServiceIdentity(AthenzService identity, + Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java index 49d9bb1ec5c..67a49059776 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.Pkcs10CsrUtils; @@ -32,8 +33,8 @@ public class InstanceRegisterInformation { @JsonProperty("token") private final boolean token; - public InstanceRegisterInformation(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceRegisterInformation(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String attestationData, Pkcs10Csr csr, boolean requestServiceToken) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java index b2af2d732bf..d1383bd04fd 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.security.Pkcs10Csr; @@ -12,7 +13,7 @@ import java.security.KeyPair; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_RSA; /** - * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzService, String, Pkcs10Csr)} + * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzIdentity, String, Pkcs10Csr)} * * @author bjorncs */ @@ -24,7 +25,7 @@ public class IdentityCsrGenerator { this.dnsSuffix = dnsSuffix; } - public Pkcs10Csr generateIdentityCsr(AthenzService identity, KeyPair keypair) { + public Pkcs10Csr generateIdentityCsr(AthenzIdentity identity, KeyPair keypair) { return Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=" + identity.getFullName()), keypair, SHA256_WITH_RSA) .addSubjectAlternativeName(String.format( "%s.%s.%s", diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java index 6b318fb16be..e5ed885b316 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identity; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import javax.net.ssl.SSLContext; @@ -13,6 +14,6 @@ import javax.net.ssl.SSLContext; * @author bjorncs */ public interface ServiceIdentityProvider { - AthenzService identity(); + AthenzIdentity identity(); SSLContext getIdentitySslContext(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index d8fa910aa73..2b0e50ed982 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identity; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyStoreType; import com.yahoo.security.SslContextBuilder; @@ -33,7 +34,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde private static final Duration REFRESH_INTERVAL = Duration.ofHours(1); private final AtomicReference<SSLContext> sslContext = new AtomicReference<>(); - private final AthenzService service; + private final AthenzIdentity service; private final File privateKeyFile; private final File certificateFile; private final File trustStoreFile; @@ -48,7 +49,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde createScheduler()); } - public SiaIdentityProvider(AthenzService service, + public SiaIdentityProvider(AthenzIdentity service, Path siaPath, File trustStoreFile) { this(service, @@ -58,7 +59,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde createScheduler()); } - public SiaIdentityProvider(AthenzService service, + public SiaIdentityProvider(AthenzIdentity service, File privateKeyFile, File certificateFile, File trustStoreFile, @@ -81,7 +82,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde } @Override - public AthenzService identity() { + public AthenzIdentity identity() { return service; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index cd35a204b00..40f12b9c6db 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; @@ -31,31 +32,31 @@ public class SiaUtils { private SiaUtils() {} - public static Path getPrivateKeyFile(AthenzService service) { + public static Path getPrivateKeyFile(AthenzIdentity service) { return getPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } - public static Path getPrivateKeyFile(Path root, AthenzService service) { + public static Path getPrivateKeyFile(Path root, AthenzIdentity service) { return root .resolve("keys") .resolve(String.format("%s.%s.key.pem", service.getDomainName(), service.getName())); } - public static Path getCertificateFile(AthenzService service) { + public static Path getCertificateFile(AthenzIdentity service) { return getCertificateFile(DEFAULT_SIA_DIRECTORY, service); } - public static Path getCertificateFile(Path root, AthenzService service) { + public static Path getCertificateFile(Path root, AthenzIdentity service) { return root .resolve("certs") .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); } - public static Optional<PrivateKey> readPrivateKeyFile(AthenzService service) { + public static Optional<PrivateKey> readPrivateKeyFile(AthenzIdentity service) { return readPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } - public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzService service) { + public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzIdentity service) { try { Path privateKeyFile = getPrivateKeyFile(root, service); if (Files.notExists(privateKeyFile)) return Optional.empty(); @@ -65,11 +66,11 @@ public class SiaUtils { } } - public static Optional<X509Certificate> readCertificateFile(AthenzService service) { + public static Optional<X509Certificate> readCertificateFile(AthenzIdentity service) { return readCertificateFile(DEFAULT_SIA_DIRECTORY, service); } - public static Optional<X509Certificate> readCertificateFile(Path root, AthenzService service) { + public static Optional<X509Certificate> readCertificateFile(Path root, AthenzIdentity service) { try { Path certificateFile = getCertificateFile(root, service); if (Files.notExists(certificateFile)) return Optional.empty(); @@ -79,11 +80,11 @@ public class SiaUtils { } } - public static void writePrivateKeyFile(AthenzService service, PrivateKey privateKey) { + public static void writePrivateKeyFile(AthenzIdentity service, PrivateKey privateKey) { writePrivateKeyFile(DEFAULT_SIA_DIRECTORY, service, privateKey); } - public static void writePrivateKeyFile(Path root, AthenzService service, PrivateKey privateKey) { + public static void writePrivateKeyFile(Path root, AthenzIdentity service, PrivateKey privateKey) { try { Path privateKeyFile = getPrivateKeyFile(root, service); Files.createDirectories(privateKeyFile.getParent()); @@ -95,11 +96,11 @@ public class SiaUtils { } } - public static void writeCertificateFile(AthenzService service, X509Certificate certificate) { + public static void writeCertificateFile(AthenzIdentity service, X509Certificate certificate) { writeCertificateFile(DEFAULT_SIA_DIRECTORY, service, certificate); } - public static void writeCertificateFile(Path root, AthenzService service, X509Certificate certificate) { + public static void writeCertificateFile(Path root, AthenzIdentity service, X509Certificate certificate) { try { Path certificateFile = getCertificateFile(root, service); Files.createDirectories(certificateFile.getParent()); @@ -111,11 +112,11 @@ public class SiaUtils { } } - public static List<AthenzService> findSiaServices() { + public static List<AthenzIdentity> findSiaServices() { return findSiaServices(DEFAULT_SIA_DIRECTORY); } - public static List<AthenzService> findSiaServices(Path root) { + public static List<AthenzIdentity> findSiaServices(Path root) { String keyFileSuffix = ".key.pem"; Path keysDirectory = root.resolve("keys"); if ( ! Files.exists(keysDirectory)) diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java index f69e937f294..0e6aff1eeca 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import org.junit.Rule; import org.junit.Test; @@ -35,7 +36,7 @@ public class SiaUtilsTest { AthenzService barService = new AthenzService("my.domain.bar"); Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, barService)); - List<AthenzService> siaIdentities = SiaUtils.findSiaServices(siaRoot); + List<AthenzIdentity> siaIdentities = SiaUtils.findSiaServices(siaRoot); assertThat(siaIdentities.size(), equalTo(2)); assertThat(siaIdentities, hasItem(fooService)); assertThat(siaIdentities, hasItem(barService)); diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index 6ee3d56673d..a33c7f15bf7 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -117,6 +117,11 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> + <jdkToolchain> + <version>${java.version}</version> + </jdkToolchain> + <source>${java.version}</source> + <target>${java.version}</target> <showDeprecation>true</showDeprecation> <compilerArgs> <arg>-Xlint:all</arg> @@ -157,18 +162,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.8.0</version> - <configuration> - <jdkToolchain> - <version>${java.version}</version> - </jdkToolchain> - <source>${java.version}</source> - <target>${java.version}</target> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index 3296cf13875..bd187ea3371 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -36,7 +36,7 @@ public final class ConnectionParams { private long connectionTimeout = TimeUnit.SECONDS.toMillis(60); private final Multimap<String, String> headers = ArrayListMultimap.create(); private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); - private int numPersistentConnectionsPerEndpoint = 8; + private int numPersistentConnectionsPerEndpoint = 1; private String proxyHost = null; private int proxyPort = 8080; private boolean useCompression = false; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 98cd13a226d..b9219d8f267 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -169,7 +169,7 @@ public class CommandLineArguments { @Option(name = {"--numPersistentConnectionsPerEndpoint"}, description = "How many tcp connections to establish per endoint.)") - private int numPersistentConnectionsPerEndpoint = 16; + private int numPersistentConnectionsPerEndpoint = 4; @Option(name = {"--maxChunkSizeBytes"}, description = "How much data to send to gateway in each message.") @@ -226,7 +226,6 @@ public class CommandLineArguments { connectionParamsBuilder .setHostnameVerifier(insecure ? NoopHostnameVerifier.INSTANCE : SSLConnectionSocketFactory.getDefaultHostnameVerifier()) - .setNumPersistentConnectionsPerEndpoint(16) .setUseCompression(useCompressionArg) .setMaxRetries(noRetryArg ? 0 : 100) .setMinTimeBetweenRetries(retrydelayArg, TimeUnit.SECONDS) diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java index 39c1257816c..bca43902b9e 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java @@ -25,7 +25,7 @@ public class ConnectionParamsTest { ConnectionParams params = new ConnectionParams.Builder().build(); assertThat(params.getHeaders().isEmpty(), is(true)); - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(8)); + assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(1)); assertThat(params.getSslContext(), nullValue()); } diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java index 53715259a0c..02509626176 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java @@ -18,7 +18,7 @@ import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.*; +import static org.junit.Assert.assertThat; public class CommandLineArgumentsTest { @@ -91,7 +91,7 @@ public class CommandLineArgumentsTest { assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080)); assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); assertThat(params.getConnectionParams().getUseCompression(), is(false)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(16)); + assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4)); assertThat(params.getFeedParams().getRoute(), is("default")); assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); @@ -106,6 +106,7 @@ public class CommandLineArgumentsTest { add("host", "hostValue"); add("port", "1234"); add("timeout", "2345"); + add("numPersistentConnectionsPerEndpoint", "7"); args.add("--useCompression"); args.add("--useDynamicThrottling"); add("maxpending", "3456"); @@ -125,6 +126,7 @@ public class CommandLineArgumentsTest { assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L)); assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456)); assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L)); + assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7)); } @Test diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 2b12c7cd78c..bfc4a611a5e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -25,7 +25,6 @@ import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vdslib.VisitorOrdering; import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.ClusterList; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import com.yahoo.yolean.concurrent.ResourceFactory; diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 6e14c3bd971..9e9fac936f4 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -88,8 +88,8 @@ public class Compressor { throw new IllegalArgumentException(requestedCompression + " is not supported"); } } - /** Compresses some data using the compression type of this compressor */ - public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(type, data, Optional.empty()); } + /** Compresses some data using the requested compression type */ + public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(requestedCompression, data, Optional.empty()); } /** Compresses some data using the compression type of this compressor */ public Compression compress(byte[] data, int uncompressedSize) { return compress(type, data, Optional.of(uncompressedSize)); } /** Compresses some data using the compression type of this compressor */ @@ -150,7 +150,7 @@ public class Compressor { * This will be either the requested compression or INCOMPRESSIBLE. */ public CompressionType type() { return compressionType; } - + /** Returns the uncompressed size of this data in bytes */ public int uncompressedSize() { return uncompressedSize; } diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java index 010a99e4ce1..f2de0ace476 100644 --- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -349,7 +349,7 @@ public abstract class IOUtils { } /** - * Reads all the content of the given array, in chunks of at max chunkSize + * Reads all the content of the given input stream, in chunks of at max chunkSize */ public static byte[] readBytes(InputStream stream, int chunkSize) throws IOException { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); diff --git a/zkfacade/src/main/sh/zkcat b/zkfacade/src/main/sh/zkcat index 675aeabbcfb..ba8924b456f 100755 --- a/zkfacade/src/main/sh/zkcat +++ b/zkfacade/src/main/sh/zkcat @@ -70,4 +70,4 @@ findhost # END environment bootstrap section -$VESPA_HOME/bin/zkctl get $@ +$VESPA_HOME/bin/vespa-zkctl get $@ diff --git a/zkfacade/src/main/sh/zkctl b/zkfacade/src/main/sh/zkctl index 721becf8c79..43112abd925 100755 --- a/zkfacade/src/main/sh/zkctl +++ b/zkfacade/src/main/sh/zkctl @@ -71,4 +71,4 @@ findhost # END environment bootstrap section # Get rid of the interactive zkcli prompt by running it in a subshell -(echo "$@" | $VESPA_HOME/bin/zkcli) +(echo "$@" | $VESPA_HOME/bin/vespa-zkcli) diff --git a/zkfacade/src/main/sh/zkls b/zkfacade/src/main/sh/zkls index c75f63efbde..5ab1204cea8 100755 --- a/zkfacade/src/main/sh/zkls +++ b/zkfacade/src/main/sh/zkls @@ -70,4 +70,4 @@ findhost # END environment bootstrap section -$VESPA_HOME/bin/zkctl ls $@ +$VESPA_HOME/bin/vespa-zkctl ls $@ |