diff options
56 files changed, 386 insertions, 429 deletions
diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh index 8f6bf953e6f..4352c8b9c0a 100755 --- a/bootstrap-cmake.sh +++ b/bootstrap-cmake.sh @@ -23,7 +23,7 @@ while getopts "uh" opt; do done shift $((OPTIND-1)) -if [[ $# -eq 0 ]]; then +if [[ $# -eq 0 ]]; then SOURCE_DIR=$(dirname "$0") EXTRA_CMAKE_ARGS="" elif [[ $# -eq 1 ]]; then diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 46acdb44746..d5e42276c1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.clustercontroller; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; @@ -10,10 +11,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.container.Container; -import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; @@ -21,6 +20,8 @@ import com.yahoo.vespa.model.container.component.Handler; import java.util.Set; import java.util.TreeSet; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + /** * Container implementation for cluster-controllers */ @@ -87,8 +88,7 @@ public class ClusterControllerContainer extends Container implements } private void addHandler(Handler h, String binding) { - h.addServerBindings("http://*/" + binding, - "https://*/" + binding); + h.addServerBindings("http://*/" + binding); super.addHandler(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 1ec72c6a2c5..09b0015540f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -101,28 +101,36 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("handled.requests.count")); - metrics.add(new Metric("handled.latency.average")); + metrics.add(new Metric("handled.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("handled.latency.max")); + metrics.add(new Metric("handled.latency.sum")); + metrics.add(new Metric("handled.latency.count")); metrics.add(new Metric("serverRejectedRequests.rate")); metrics.add(new Metric("serverRejectedRequests.count")); - metrics.add(new Metric("serverThreadPoolSize.average")); - metrics.add(new Metric("serverThreadPoolSize.min")); - metrics.add(new Metric("serverThreadPoolSize.max")); - metrics.add(new Metric("serverThreadPoolSize.rate")); - metrics.add(new Metric("serverThreadPoolSize.count")); + metrics.add(new Metric("serverThreadPoolSize.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.min")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.max")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.rate")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("serverThreadPoolSize.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverThreadPoolSize.last")); - metrics.add(new Metric("serverActiveThreads.average")); + metrics.add(new Metric("serverActiveThreads.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverActiveThreads.min")); metrics.add(new Metric("serverActiveThreads.max")); - metrics.add(new Metric("serverActiveThreads.rate")); + metrics.add(new Metric("serverActiveThreads.rate")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); - metrics.add(new Metric("httpapi_latency.average")); - metrics.add(new Metric("httpapi_pending.average")); + metrics.add(new Metric("httpapi_latency.max")); + metrics.add(new Metric("httpapi_latency.sum")); + metrics.add(new Metric("httpapi_latency.count")); + metrics.add(new Metric("httpapi_latency.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("httpapi_pending.max")); + metrics.add(new Metric("httpapi_pending.sum")); + metrics.add(new Metric("httpapi_pending.count")); + metrics.add(new Metric("httpapi_pending.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("httpapi_num_operations.rate")); metrics.add(new Metric("httpapi_num_updates.rate")); metrics.add(new Metric("httpapi_num_removes.rate")); @@ -133,6 +141,7 @@ public class VespaMetricSet { metrics.add(new Metric("mem.heap.total.average")); metrics.add(new Metric("mem.heap.free.average")); metrics.add(new Metric("mem.heap.used.average")); + metrics.add(new Metric("mem.heap.used.max")); metrics.add(new Metric("jdisc.memory_mappings.max")); metrics.add(new Metric("jdisc.open_file_descriptors.max")); @@ -158,10 +167,14 @@ public class VespaMetricSet { metrics.add(new Metric("http.status.401.rate")); metrics.add(new Metric("http.status.403.rate")); - metrics.add(new Metric("jdisc.http.request.uri_length.average")); metrics.add(new Metric("jdisc.http.request.uri_length.max")); - metrics.add(new Metric("jdisc.http.request.content_size.average")); + metrics.add(new Metric("jdisc.http.request.uri_length.sum")); + metrics.add(new Metric("jdisc.http.request.uri_length.count")); + metrics.add(new Metric("jdisc.http.request.uri_length.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("jdisc.http.request.content_size.max")); + metrics.add(new Metric("jdisc.http.request.content_size.sum")); + metrics.add(new Metric("jdisc.http.request.content_size.count")); + metrics.add(new Metric("jdisc.http.request.content_size.average")); // TODO: Remove in Vespa 8 return metrics; } @@ -197,36 +210,59 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); metrics.add(new Metric("peak_qps.max")); - metrics.add(new Metric("search_connections.average")); - metrics.add(new Metric("active_queries.average")); - metrics.add(new Metric("feed.latency.average")); + metrics.add(new Metric("search_connections.max")); + metrics.add(new Metric("search_connections.sum")); + metrics.add(new Metric("search_connections.count")); + metrics.add(new Metric("search_connections.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("active_queries.max")); + metrics.add(new Metric("active_queries.sum")); + metrics.add(new Metric("active_queries.count")); + metrics.add(new Metric("active_queries.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("feed.latency.max")); + metrics.add(new Metric("feed.latency.sum")); + metrics.add(new Metric("feed.latency.count")); + metrics.add(new Metric("feed.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("feed.http-requests.count")); metrics.add(new Metric("feed.http-requests.rate")); metrics.add(new Metric("queries.rate")); - metrics.add(new Metric("query_container_latency.average")); - metrics.add(new Metric("query_latency.average")); + metrics.add(new Metric("query_container_latency.max")); + metrics.add(new Metric("query_container_latency.sum")); + metrics.add(new Metric("query_container_latency.count")); + metrics.add(new Metric("query_container_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("query_latency.max")); + metrics.add(new Metric("query_latency.sum")); + metrics.add(new Metric("query_latency.count")); + metrics.add(new Metric("query_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("query_latency.95percentile")); metrics.add(new Metric("query_latency.99percentile")); metrics.add(new Metric("failed_queries.rate")); metrics.add(new Metric("degraded_queries.rate")); - metrics.add(new Metric("hits_per_query.average")); + metrics.add(new Metric("hits_per_query.max")); + metrics.add(new Metric("hits_per_query.sum")); + metrics.add(new Metric("hits_per_query.count")); + metrics.add(new Metric("hits_per_query.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("documents_covered.count")); metrics.add(new Metric("documents_total.count")); metrics.add(new Metric("dispatch_internal.rate")); metrics.add(new Metric("dispatch_fdispatch.rate")); - metrics.add(new Metric("totalhits_per_query.average")); + metrics.add(new Metric("totalhits_per_query.max")); + metrics.add(new Metric("totalhits_per_query.sum")); + metrics.add(new Metric("totalhits_per_query.count")); + metrics.add(new Metric("totalhits_per_query.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("empty_results.rate")); metrics.add(new Metric("requestsOverQuota.rate")); metrics.add(new Metric("requestsOverQuota.count")); - metrics.add(new Metric("relevance.at_1.average")); + metrics.add(new Metric("relevance.at_1.sum")); metrics.add(new Metric("relevance.at_1.count")); - metrics.add(new Metric("relevance.at_3.average")); + metrics.add(new Metric("relevance.at_1.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("relevance.at_3.sum")); metrics.add(new Metric("relevance.at_3.count")); - metrics.add(new Metric("relevance.at_10.average")); + metrics.add(new Metric("relevance.at_3.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("relevance.at_10.sum")); metrics.add(new Metric("relevance.at_10.count")); + metrics.add(new Metric("relevance.at_10.average")); // TODO: Remove in Vespa 8 // Errors from qrserver metrics.add(new Metric("error.timeout.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index a370e0dc38b..446169c0ed2 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -212,9 +212,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT, - "https://*" + StateHandler.STATE_API_ROOT, - "http://*" + StateHandler.STATE_API_ROOT + "/*", - "https://*" + StateHandler.STATE_API_ROOT + "/*"); + "http://*" + StateHandler.STATE_API_ROOT + "/*"); addComponent(stateHandler); } @@ -242,13 +240,13 @@ public abstract class ContainerCluster<CONTAINER extends Container> Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( APPLICATION_STATUS_HANDLER_CLASS, null), null)); - statusHandler.addServerBindings("http://*/ApplicationStatus", "https://*/ApplicationStatus"); + statusHandler.addServerBindings("http://*/ApplicationStatus"); addComponent(statusHandler); } public void addVipHandler() { Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); - vipHandler.addServerBindings("http://*/status.html", "https://*/status.html"); + vipHandler.addServerBindings("http://*/status.html"); addComponent(vipHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index b7c8460a9a5..6b4f8d486ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -49,9 +49,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra public static Handler<?> getHandler() { Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); handler.addServerBindings("http://*/" + REST_BINDING, - "https://*/" + REST_BINDING, - "http://*/" + REST_BINDING + "/*", - "https://*/" + REST_BINDING + "/*"); + "http://*/" + REST_BINDING + "/*"); return handler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 4220a6571a0..071411845ad 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -153,6 +153,6 @@ public final class AccessControl { } private static Stream<String> servletBindings(Servlet servlet) { - return Stream.of("http://*/", "https://*/").map(protocol -> protocol + servlet.bindingPath); + return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java index 32f0f373a92..4fd79a4f335 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java @@ -4,10 +4,6 @@ package com.yahoo.vespa.model.container.processing; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.component.chain.Chains; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - /** * Root config producer for processing * @@ -15,7 +11,7 @@ import java.util.List; */ public class ProcessingChains extends Chains<ProcessingChain> { - public static final String[] defaultBindings = new String[] {"http://*/processing/*", "https://*/processing/*"}; + public static final String[] defaultBindings = new String[] {"http://*/processing/*"}; public ProcessingChains(AbstractConfigProducer parent, String subId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 6663dc0bbc0..7e80c6be221 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -271,7 +271,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_SETTING)); cluster.addComponent( new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), - "http://*/" + name, "https://*/" + name)); + "http://*/" + name)); } else { cluster.addVipHandler(); } @@ -712,7 +712,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); - String[] defaultBindings = {"http://*/search/*", "https://*/search/*"}; + String[] defaultBindings = {"http://*/search/*"}; for (String binding: serverBindings(searchElement, defaultBindings)) { searchHandler.addServerBindings(binding); } @@ -722,7 +722,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addGUIHandler(ApplicationContainerCluster cluster) { Handler<?> guiHandler = new GUIHandler(); - guiHandler.addServerBindings("http://"+GUIHandler.BINDING, "https://"+GUIHandler.BINDING); + guiHandler.addServerBindings("http://"+GUIHandler.BINDING); cluster.addComponent(guiHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index a44f5440a9f..10e24a609f7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -18,7 +18,7 @@ import java.util.logging.Logger; public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); - private static final String[] DEFAULT_BINDINGS = {"http://*/", "https://*/"}; + private static final String[] DEFAULT_BINDINGS = {"http://*/"}; public static ContainerDocumentApi.Options build(Element spec) { return new ContainerDocumentApi.Options( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index fd1ea2ae7ab..f5df3df0070 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -196,8 +196,8 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { @Test public void servlet_can_be_excluded_by_excluding_one_of_its_bindings() throws Exception { final String servletPath = "servlet/path"; - final String notExcludedBinding = "https://*/" + servletPath; - final String excludedBinding = "http://*/" + servletPath; + final String notExcludedBinding = "http://*:8081/" + servletPath; + final String excludedBinding = "http://*:8080/" + servletPath; Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", httpWithExcludedBinding(excludedBinding), @@ -217,8 +217,8 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { @Test public void rest_api_can_be_excluded_by_excluding_one_of_its_bindings() throws Exception { final String restApiPath = "api/v0"; - final String notExcludedBinding = "http://*/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; - final String excludedBinding = "https://*/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; + final String notExcludedBinding = "http://*:8081/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; + final String excludedBinding = "http://*:8080/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; Element clusterElem = DomBuilderTest.parse( "<jdisc version='1.0'>", httpWithExcludedBinding(excludedBinding), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index 428286c4794..5b92934678d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -16,7 +16,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertThat; @@ -64,7 +63,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa Element elem = DomBuilderTest.parse( "<jdisc id='cluster1' version='1.0'>", " <document-api>", - " <binding>https://*/document-api/</binding>", + " <binding>http://*/document-api/</binding>", " <binding>missing-trailing-slash</binding>", " </document-api>", nodesXml, @@ -77,8 +76,8 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa private void verifyCustomBindings(String id, String bindingSuffix) { Handler<?> handler = getHandlers("cluster1").get(id); - assertThat(handler.getServerBindings(), hasItem("https://*/document-api/" + bindingSuffix)); - assertThat(handler.getServerBindings(), hasItem("https://*/document-api/" + bindingSuffix + "/")); + assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix)); + assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix + "/")); assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix)); assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix + "/")); @@ -102,9 +101,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(4)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(2)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 26820672f2c..31077df7f7c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -35,7 +35,6 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.component.HttpFilter; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Test; @@ -266,11 +265,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName()); assertThat(applicationStatusHandler.serverBindings(), - contains("http://*/ApplicationStatus", "https://*/ApplicationStatus")); + contains("http://*/ApplicationStatus")); JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName()); assertThat(fileRequestHandler.serverBindings(), - contains("http://*/status.html", "https://*/status.html")); + contains("http://*/status.html")); } @Test diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 40a2b31b3fc..d5a02949cf3 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -75,7 +75,6 @@ <access-control domain="my.athens-domain" read="true"> <exclude> <binding>http//*/foo/*</binding> - <binding>https://*/foo/*</binding> </exclude> <application>my-app</application> <vespa-domain>vespa.vespa.cd</vespa-domain> @@ -134,7 +133,6 @@ <document-api> <binding>http://*/document-api/</binding> - <binding>https://*/document-api/</binding> <abortondocumenterror>false</abortondocumenterror> <retryenabled>false</retryenabled> <timeout>5.55</timeout> diff --git a/config-model/src/test/schema-test-files/standalone-container.xml b/config-model/src/test/schema-test-files/standalone-container.xml index e36218fa084..0656f293001 100644 --- a/config-model/src/test/schema-test-files/standalone-container.xml +++ b/config-model/src/test/schema-test-files/standalone-container.xml @@ -70,7 +70,6 @@ <document-api> <binding>http://*/document-api/</binding> - <binding>https://*/document-api/</binding> <abortondocumenterror>false</abortondocumenterror> <retryenabled>false</retryenabled> <timeout>5.55</timeout> diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index accc313dcae..4c98ca64847 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(configserver) +install_configserver_component(configserver) vespa_install_script(src/main/sh/vespa-configserver-remove-state bin) vespa_install_script(src/main/sh/ping-configserver libexec/vespa) @@ -10,16 +10,15 @@ vespa_install_script(src/main/sh/stop-configserver libexec/vespa) install(DIRECTORY src/main/resources/logd DESTINATION conf) install(DIRECTORY src/main/resources/configserver-app DESTINATION conf) -install(CODE "execute_process(COMMAND mkdir -p \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components)") -install(CODE "execute_process(COMMAND mkdir -p \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/config-models)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/athenz-identity-provider-service.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/config-model-fat.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/config-model-fat.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-flags-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver-flags.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/flags-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/flags.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/orchestrator-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/orchestrator.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/service-monitor-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/service-monitor.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/application-model-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/application-model.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/node-repository-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/node-repository.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/lib/jars/zkfacade-jar-with-dependencies.jar \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/zkfacade.jar)") -install(CODE "execute_process(COMMAND ln -snf \${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/lib/jars/config-models)") +install(DIRECTORY DESTINATION conf/configserver-app/components) +install(DIRECTORY DESTINATION conf/configserver-app/config-models) +install_symlink(lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar conf/configserver-app/components/athenz-identity-provider-service.jar) +install_symlink(lib/jars/config-model-fat.jar conf/configserver-app/components/config-model-fat.jar) +install_symlink(lib/jars/configserver-flags-jar-with-dependencies.jar conf/configserver-app/components/configserver-flags.jar) +install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/configserver-app/components/flags.jar) +install_symlink(lib/jars/orchestrator-jar-with-dependencies.jar conf/configserver-app/components/orchestrator.jar) +install_symlink(lib/jars/service-monitor-jar-with-dependencies.jar conf/configserver-app/components/service-monitor.jar) +install_symlink(lib/jars/application-model-jar-with-dependencies.jar conf/configserver-app/components/application-model.jar) +install_symlink(lib/jars/node-repository-jar-with-dependencies.jar conf/configserver-app/components/node-repository.jar) +install_symlink(lib/jars/zkfacade-jar-with-dependencies.jar conf/configserver-app/components/zkfacade.jar) +install_symlink(conf/configserver-app/components lib/jars/config-models) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index c4b1d0f7c86..53d20379989 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -301,12 +301,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) return false; TenantApplications tenantApplications = tenant.getApplicationRepo(); - if (!tenantApplications.listApplications().contains(applicationId)) return false; + if (!tenantApplications.activeApplications().contains(applicationId)) return false; // Deleting an application is done by deleting the remote session and waiting // until the config server where the deployment happened picks it up and deletes // the local session - long sessionId = tenantApplications.getSessionIdForApplication(applicationId); + long sessionId = tenantApplications.requireActiveSessionOf(applicationId); RemoteSession remoteSession = getRemoteSession(tenant, sessionId); remoteSession.createDeleteTransaction().commit(); @@ -324,7 +324,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server // this is done asynchronously in application maintenance by the node repository) - transaction.add(tenantApplications.deleteApplication(applicationId)); + transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); @@ -425,7 +425,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<ApplicationId> listApplications() { return tenantRepository.getAllTenants().stream() - .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream()) + .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream()) .collect(Collectors.toSet()); } @@ -483,7 +483,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (applicationRepo == null) throw new IllegalArgumentException("Application repo for tenant '" + tenant.getName() + "' not found"); - return applicationRepo.getSessionIdForApplication(applicationId); + return applicationRepo.requireActiveSessionOf(applicationId); } public void validateThatRemoteSessionIsNotActive(Tenant tenant, long sessionId) { @@ -565,7 +565,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private List<ApplicationId> activeApplications(TenantName tenantName) { - return tenantRepository.getTenant(tenantName).getApplicationRepo().listApplications(); + return tenantRepository.getTenant(tenantName).getApplicationRepo().activeApplications(); } // ---------------- Misc operations ---------------------------------------------------------------- @@ -616,7 +616,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); TenantApplications applicationRepo = tenant.getApplicationRepo(); try { - long currentActiveSessionId = applicationRepo.getSessionIdForApplication(appId); + long currentActiveSessionId = applicationRepo.requireActiveSessionOf(appId); RemoteSession currentActiveSession = getRemoteSession(tenant, currentActiveSessionId); if (currentActiveSession != null) { currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); @@ -646,7 +646,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<ApplicationId> listApplicationIds(Tenant tenant) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.listApplications(); + return applicationRepo.activeApplications(); } private void cleanupTempDirectory(File tempDir) { @@ -658,13 +658,13 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private LocalSession getExistingSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return getLocalSession(tenant, applicationRepo.getSessionIdForApplication(applicationId)); + return getLocalSession(tenant, applicationRepo.requireActiveSessionOf(applicationId)); } private LocalSession getActiveSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - if (applicationRepo.listApplications().contains(applicationId)) { - return tenant.getLocalSessionRepo().getSession(applicationRepo.getSessionIdForApplication(applicationId)); + if (applicationRepo.activeApplications().contains(applicationId)) { + return tenant.getLocalSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 3705a0ec145..656030bb8d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -17,18 +17,19 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import java.util.List; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * The applications of a tenant, backed by ZooKeeper. * - * Each application is stored as a single node under /config/v2/tenants/<tenant>/applications/<applications>, - * named the same as the application id and containing the id of the session storing the content of the application. + * Each application is stored under /config/v2/tenants/<tenant>/applications/<applications>, + * the root contains the currently active session, if any, and sub-paths /preparing contains the session id + * of whatever session may be activated next, if any, and /lock is used for synchronizing writes to all these paths. * * @author Ulf Lilleengen */ @@ -69,16 +70,18 @@ public class TenantApplications { * * @return a list of {@link ApplicationId}s that are active. */ - public List<ApplicationId> listApplications() { + public List<ApplicationId> activeApplications() { return curator.getChildren(applicationsPath).stream() - .flatMap(this::parseApplication) + .filter(this::isValid) + .map(ApplicationId::fromSerializedForm) + .filter(id -> activeSessionOf(id).isPresent()) .collect(Collectors.toUnmodifiableList()); } - // TODO jvenstad: Remove after it has run once everywhere. - private Stream<ApplicationId> parseApplication(String appNode) { + private boolean isValid(String appNode) { // TODO jvenstad: Remove after it has run once everywhere. try { - return Stream.of(ApplicationId.fromSerializedForm(appNode)); + ApplicationId.fromSerializedForm(appNode); + return true; } catch (IllegalArgumentException __) { log.log(LogLevel.INFO, TenantRepository.logPre(tenant) + "Unable to parse application id from '" + appNode + "'; deleting it as it shouldn't be here."); @@ -88,18 +91,25 @@ public class TenantApplications { catch (Exception e) { log.log(LogLevel.WARNING, TenantRepository.logPre(tenant) + "Failed to clean up stray node '" + appNode + "'!", e); } - return Stream.empty(); + return false; } } + /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */ + public OptionalLong activeSessionOf(ApplicationId id) { + String data = curator.getData(applicationPath(id)).map(Utf8::toString) + .orElseThrow(() -> new IllegalArgumentException("Unknown application '" + id + "'.")); + return data.isEmpty() ? OptionalLong.empty() : OptionalLong.of(Long.parseLong(data)); + } + /** * Returns a transaction which writes the given session id as the currently active for the given application. * * @param applicationId An {@link ApplicationId} that represents an active application. * @param sessionId Id of the session containing the application package for this id. */ - public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { - if (listApplications().contains(applicationId)) { + public Transaction createPutTransaction(ApplicationId applicationId, long sessionId) { + if (curator.exists(applicationPath(applicationId))) { return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); } else { return new CuratorTransaction(curator).add(CuratorOperations.create(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); @@ -107,25 +117,21 @@ public class TenantApplications { } /** - * Return the stored session id for a given application. + * Return the active session id for a given application. * * @param applicationId an {@link ApplicationId} * @return session id of given application id. * @throws IllegalArgumentException if the application does not exist */ - public long getSessionIdForApplication(ApplicationId applicationId) { - String path = applicationPath(applicationId).getAbsolute(); - try { - return Long.parseLong(Utf8.toString(curator.framework().getData().forPath(path))); - } catch (Exception e) { - throw new IllegalArgumentException(TenantRepository.logPre(applicationId) + "Unable to read the session id from '" + path + "'", e); - } + public long requireActiveSessionOf(ApplicationId applicationId) { + return activeSessionOf(applicationId) + .orElseThrow(() -> new IllegalArgumentException("Application '" + applicationId + "' has no active session.")); } /** * Returns a transaction which deletes this application. */ - public CuratorTransaction deleteApplication(ApplicationId applicationId) { + public CuratorTransaction createDeleteTransaction(ApplicationId applicationId) { return CuratorTransaction.from(CuratorOperations.delete(applicationPath(applicationId).getAbsolute()), curator); } @@ -133,7 +139,7 @@ public class TenantApplications { * Removes all applications not known to this from the config server state. */ public void removeUnusedApplications() { - reloadHandler.removeApplicationsExcept(Set.copyOf(listApplications())); + reloadHandler.removeApplicationsExcept(Set.copyOf(activeApplications())); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index 6345532d4ff..cdf995b80bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -56,7 +56,7 @@ public class ListApplicationsHandler extends HttpHandler { Utils.checkThatTenantExists(tenantRepository, tenantName); Tenant tenant = tenantRepository.getTenant(tenantName); TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.listApplications(); + return applicationRepo.activeApplications(); } private static String createUrlStringFromId(String urlBase, ApplicationId id, Zone zone) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 0cdf5ebfe95..af8956803ab 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -95,7 +95,7 @@ public class LocalSession extends Session implements Comparable<LocalSession> { zooKeeperClient.createActiveWaiter(); superModelGenerationCounter.increment(); // TODO jvenstad: I hope this counter isn't used for serious things, as it's updated way ahead of activation. Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutApplicationTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); + transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 15182813a22..ccd5684b9ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -190,9 +190,9 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { } private void loadSessionIfActive(RemoteSession session) { - for (ApplicationId applicationId : applicationRepo.listApplications()) { + for (ApplicationId applicationId : applicationRepo.activeApplications()) { try { - if (applicationRepo.getSessionIdForApplication(applicationId) == session.getSessionId()) { + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { log.log(LogLevel.DEBUG, "Found active application for session " + session.getSessionId() + " , loading it"); loadActiveSession(session); break; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 90eeb89dc8e..d8ba5890545 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -188,9 +188,9 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { } private long getActiveSessionId(ApplicationId applicationId) { - List<ApplicationId> applicationIds = applicationRepo.listApplications(); + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); if (applicationIds.contains(applicationId)) { - return applicationRepo.getSessionIdForApplication(applicationId); + return applicationRepo.requireActiveSessionOf(applicationId); } return nonExistingActiveSession; } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index b2c426e707b..5aca82409a9 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -76,115 +76,77 @@ <handler id='com.yahoo.vespa.config.server.http.HttpGetConfigHandler' bundle='configserver'> <binding>http://*/config/v1/*/*</binding> - <binding>https://*/config/v1/*/*</binding> <binding>http://*/config/v1/*</binding> - <binding>https://*/config/v1/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.HttpListConfigsHandler' bundle='configserver'> <binding>http://*/config/v1/</binding> - <binding>https://*/config/v1/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.HttpListNamedConfigsHandler' bundle='configserver'> <binding>http://*/config/v1/*/</binding> - <binding>https://*/config/v1/*/</binding> <binding>http://*/config/v1/*/*/</binding> - <binding>https://*/config/v1/*/*/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.status.StatusHandler' bundle='configserver'> <binding>http://*/status</binding> - <binding>https://*/status</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.flags.FlagsHandler' bundle='configserver'> <binding>http://*/flags/v1</binding> - <binding>https://*/flags/v1</binding> <binding>http://*/flags/v1/*</binding> - <binding>https://*/flags/v1/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.TenantHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/</binding> - <binding>https://*/application/v2/tenant/</binding> <binding>http://*/application/v2/tenant/*</binding> - <binding>https://*/application/v2/tenant/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionCreateHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session</binding> - <binding>https://*/application/v2/tenant/*/session</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionPrepareHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/prepared</binding> - <binding>https://*/application/v2/tenant/*/session/*/prepared</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionActiveHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/active</binding> - <binding>https://*/application/v2/tenant/*/session/*/active</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/prepareandactivate</binding> - <binding>https://*/application/v2/tenant/*/prepareandactivate</binding> <binding>http://*/application/v2/tenant/*/session/*/prepareandactivate</binding> - <binding>https://*/application/v2/tenant/*/session/*/prepareandactivate</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.SessionContentHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/session/*/content/*</binding> - <binding>https://*/application/v2/tenant/*/session/*/content/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ListApplicationsHandler' bundle='configserver'> <binding>http://*/application/v2/tenant/*/application/</binding> - <binding>https://*/application/v2/tenant/*/application/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.ApplicationHandler' bundle='configserver'> <!-- WARNING: THIS LIST *MUST* MATCH THE ONE IN ApplicationHandler::getBindingMatch --> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*</binding> <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*</binding> - <binding>https://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*</binding> <binding>http://*/application/v2/tenant/*/application/*</binding> - <binding>https://*/application/v2/tenant/*/application/*</binding> <binding>http://*/application/v2/tenant/*/application/*/logs</binding> - <binding>https://*/application/v2/tenant/*/application/*/logs</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HttpGetConfigHandler' bundle='configserver'> <binding>http://*/config/v2/tenant/*/application/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/*</binding> <binding>http://*/config/v2/tenant/*/application/*/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/*/*</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler' bundle='configserver'> <binding>http://*/config/v2/tenant/*/application/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HttpListNamedConfigsHandler' bundle='configserver'> <binding>http://*/config/v2/tenant/*/application/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/*/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/</binding> <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/</binding> - <binding>https://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/</binding> </handler> <handler id='com.yahoo.vespa.config.server.http.v2.HostHandler' bundle='configserver'> <binding>http://*/application/v2/host/*</binding> - <binding>https://*/application/v2/host/*</binding> </handler> <http> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 5d356ddf88e..e0fa760b35d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -315,7 +315,7 @@ public class ApplicationRepositoryTest { assertTrue(deployment2.isPresent()); deployment2.get().activate(); // session 3 - long activeSessionId = tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId()); + long activeSessionId = tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId()); clock.advance(Duration.ofSeconds(10)); Optional<com.yahoo.config.provision.Deployment> deployment3 = tester.redeployFromLocalActive(); @@ -325,7 +325,7 @@ public class ApplicationRepositoryTest { LocalSession deployment3session = ((com.yahoo.vespa.config.server.deploy.Deployment) deployment3.get()).session(); assertNotEquals(activeSessionId, deployment3session); // No change to active session id - assertEquals(activeSessionId, tester.tenant().getApplicationRepo().getSessionIdForApplication(tester.applicationId())); + assertEquals(activeSessionId, tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId())); assertEquals(3, tester.tenant().getLocalSessionRepo().listSessions().size()); clock.advance(Duration.ofHours(1)); // longer than session lifetime diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index a708e4d8ace..01a7d5e0239 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -39,12 +39,12 @@ public class TenantApplicationsTest { writeApplicationData(createApplicationId("foo"), 3L); writeApplicationData(createApplicationId("bar"), 4L); TenantApplications repo = createZKAppRepo(); - List<ApplicationId> applications = repo.listApplications(); + List<ApplicationId> applications = repo.activeApplications(); assertThat(applications.size(), is(2)); assertThat(applications.get(0).application().value(), is("foo")); assertThat(applications.get(1).application().value(), is("bar")); - assertThat(repo.getSessionIdForApplication(applications.get(0)), is(3L)); - assertThat(repo.getSessionIdForApplication(applications.get(1)), is(4L)); + assertThat(repo.requireActiveSessionOf(applications.get(0)), is(3L)); + assertThat(repo.requireActiveSessionOf(applications.get(1)), is(4L)); } @Test @@ -52,7 +52,7 @@ public class TenantApplicationsTest { writeApplicationData(createApplicationId("foo"), 3L); writeApplicationData("invalid", 3L); TenantApplications repo = createZKAppRepo(); - List<ApplicationId> applications = repo.listApplications(); + List<ApplicationId> applications = repo.activeApplications(); assertThat(applications.size(), is(1)); assertThat(applications.get(0).application().value(), is("foo")); } @@ -60,7 +60,7 @@ public class TenantApplicationsTest { @Test(expected = IllegalArgumentException.class) public void require_that_requesting_session_for_unknown_application_throws_exception() throws Exception { TenantApplications repo = createZKAppRepo(); - repo.getSessionIdForApplication(createApplicationId("nonexistent")); + repo.requireActiveSessionOf(createApplicationId("nonexistent")); } @Test(expected = IllegalArgumentException.class) @@ -70,18 +70,18 @@ public class TenantApplicationsTest { curatorFramework.create().creatingParentsIfNeeded() .forPath(TenantRepository.getApplicationsPath(tenantName).append(baz.serializedForm()).getAbsolute()); TenantApplications repo = createZKAppRepo(); - repo.getSessionIdForApplication(baz); + repo.requireActiveSessionOf(baz); } @Test public void require_that_application_ids_can_be_written() throws Exception { TenantApplications repo = createZKAppRepo(); ApplicationId myapp = createApplicationId("myapp"); - repo.createPutApplicationTransaction(myapp, 3l).commit(); + repo.createPutTransaction(myapp, 3l).commit(); String path = TenantRepository.getApplicationsPath(tenantName).append(myapp.serializedForm()).getAbsolute(); assertTrue(curatorFramework.checkExists().forPath(path) != null); assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("3")); - repo.createPutApplicationTransaction(myapp, 5l).commit(); + repo.createPutTransaction(myapp, 5l).commit(); assertTrue(curatorFramework.checkExists().forPath(path) != null); assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("5")); } @@ -91,13 +91,13 @@ public class TenantApplicationsTest { TenantApplications repo = createZKAppRepo(); ApplicationId id1 = createApplicationId("myapp"); ApplicationId id2 = createApplicationId("myapp2"); - repo.createPutApplicationTransaction(id1, 1).commit(); - repo.createPutApplicationTransaction(id2, 1).commit(); - assertThat(repo.listApplications().size(), is(2)); - repo.deleteApplication(id1).commit(); - assertThat(repo.listApplications().size(), is(1)); - repo.deleteApplication(id2).commit(); - assertThat(repo.listApplications().size(), is(0)); + repo.createPutTransaction(id1, 1).commit(); + repo.createPutTransaction(id2, 1).commit(); + assertThat(repo.activeApplications().size(), is(2)); + repo.createDeleteTransaction(id1).commit(); + assertThat(repo.activeApplications().size(), is(1)); + repo.createDeleteTransaction(id2).commit(); + assertThat(repo.activeApplications().size(), is(0)); } @Test @@ -108,7 +108,7 @@ public class TenantApplicationsTest { MockReloadHandler reloadHandler = new MockReloadHandler(); TenantApplications repo = createZKAppRepo(reloadHandler); assertNull(reloadHandler.lastRemoved); - repo.deleteApplication(foo).commit(); + repo.createDeleteTransaction(foo).commit(); long endTime = System.currentTimeMillis() + 60_000; while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { Thread.sleep(100); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 7ffb9552cf8..6b67dcc4e9a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -183,7 +183,7 @@ public class DeployTester { public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() - .getSessionIdForApplication(applicationId)); + .requireActiveSessionOf(applicationId)); return session.getAllocatedHosts(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index b0bb3bf244f..3d34a4eeaf5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -56,12 +56,12 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { session2 = new MockSession(2l, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); Tenant tenant1 = tenantRepository.getTenant(tenantName1); tenant1.getLocalSessionRepo().addSession(session2); - tenant1.getApplicationRepo().createPutApplicationTransaction(idTenant1, 2l).commit(); + tenant1.getApplicationRepo().createPutTransaction(idTenant1, 2l).commit(); MockSession session3 = new MockSession(3l, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); Tenant tenant2 = tenantRepository.getTenant(tenantName2); tenant2.getLocalSessionRepo().addSession(session3); - tenant2.getApplicationRepo().createPutApplicationTransaction(idTenant2, 3l).commit(); + tenant2.getApplicationRepo().createPutTransaction(idTenant2, 3l).commit(); handler = new ApplicationHandler(ApplicationHandler.testOnlyContext(), Zone.defaultZone(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index d3dcbcc2252..fa560ceae4d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -223,13 +223,13 @@ public class ApplicationHandlerTest { } private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { - long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().getSessionIdForApplication(applicationId); + long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); assertNull(tenantRepository.getTenant(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); } private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { - long sessionId = tenant.getApplicationRepo().getSessionIdForApplication(applicationId); + long sessionId = tenant.getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, true); assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 2d3dcc592f7..fb75e91dfd6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -46,7 +46,7 @@ public class HostHandlerTest { private HostHandler hostHandler; static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId) { - tenant.getApplicationRepo().createPutApplicationTransaction(applicationId, sessionId).commit(); + tenant.getApplicationRepo().createPutTransaction(applicationId, sessionId).commit(); ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index f57e7f09b39..b34f7a0c487 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -51,12 +51,12 @@ public class ListApplicationsHandlerTest { final String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; assertResponse(url, Response.Status.OK, "[]"); - applicationRepo.createPutApplicationTransaction( + applicationRepo.createPutTransaction( new ApplicationId.Builder().tenant("tenant").applicationName("foo").instanceName("quux").build(), 1).commit(); assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); - applicationRepo.createPutApplicationTransaction( + applicationRepo.createPutTransaction( new ApplicationId.Builder().tenant("tenant").applicationName("bali").instanceName("quux").build(), 1).commit(); assertResponse(url, Response.Status.OK, @@ -82,10 +82,10 @@ public class ListApplicationsHandlerTest { @Test public void require_that_listing_works_with_multiple_tenants() throws Exception { - applicationRepo.createPutApplicationTransaction(new ApplicationId.Builder() + applicationRepo.createPutTransaction(new ApplicationId.Builder() .tenant("tenant") .applicationName("foo").instanceName("quux").build(), 1).commit(); - applicationRepo2.createPutApplicationTransaction(new ApplicationId.Builder() + applicationRepo2.createPutTransaction(new ApplicationId.Builder() .tenant("tenant") .applicationName("quux").instanceName("foo").build(), 1).commit(); String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 927ac26b77d..bc509fcd802 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -194,7 +194,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .applicationName("foo") .instanceName("quux") .build(); - applicationRepo.createPutApplicationTransaction(fooId, 2).commit(); + applicationRepo.createPutTransaction(fooId, 2).commit(); assertFromParameter("3", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux"); localSessionRepo.addSession(new SessionHandlerTest.MockSession(5l, FilesApplicationPackage.fromFile(testApp))); ApplicationId bioId = new ApplicationId.Builder() @@ -202,7 +202,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .applicationName("foobio") .instanceName("quux") .build(); - applicationRepo.createPutApplicationTransaction(bioId, 5).commit(); + applicationRepo.createPutTransaction(bioId, 5).commit(); assertFromParameter("6", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foobio/environment/staging/region/baz/instance/quux"); } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index f6bdeb02b9c..645c231531d 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -6,12 +6,8 @@ import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import org.json.JSONException; -import org.json.JSONObject; -import java.io.IOException; import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.time.Instant; import java.util.Optional; import java.util.concurrent.Executor; @@ -32,47 +28,19 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - JSONObject responseJSON = new JSONObject(); Instant earliestLogThreshold = Optional.ofNullable(request.getProperty("from")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElse(Instant.MIN); Instant latestLogThreshold = Optional.ofNullable(request.getProperty("to")) .map(Long::valueOf).map(Instant::ofEpochMilli).orElseGet(Instant::now); - try { - if (request.hasProperty("streaming")) { - return new HttpResponse(200) { - { - headers().add("Content-Encoding", "gzip"); - } - - @Override - public void render(OutputStream outputStream) { - logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); - } - }; - } - - JSONObject logJson = logReader.readLogs(earliestLogThreshold, latestLogThreshold); - responseJSON.put("logs", logJson); - } catch (IOException | JSONException e) { - return new HttpResponse(404) { - @Override - public void render(OutputStream outputStream) {} - }; - } - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream); - outputStreamWriter.write(responseJSON.toString()); - outputStreamWriter.close(); + { + headers().add("Content-Encoding", "gzip"); } - @Override - public String getContentType() { - return "application/json"; + public void render(OutputStream outputStream) { + logReader.writeLogs(outputStream, earliestLogThreshold, latestLogThreshold); } }; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index 663741f9bef..6cb92244522 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -3,10 +3,7 @@ package com.yahoo.container.handler; import com.yahoo.collections.Pair; import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; @@ -16,15 +13,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.time.Duration; import java.time.Instant; -import java.util.Base64; import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; class LogReader { @@ -41,19 +35,6 @@ class LogReader { this.logFilePattern = logFilePattern; } - // TODO: Remove when all clients use streaming - JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws IOException, JSONException { - JSONObject json = new JSONObject(); - latestLogThreshold = latestLogThreshold.plus(Duration.ofMinutes(5)); // Add some time to allow retrieving logs currently being modified - for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { - StringBuilder filenameBuilder = new StringBuilder(); - logDirectory.relativize(file).iterator().forEachRemaining(p -> filenameBuilder.append("-").append(p.getFileName().toString())); - byte[] fileData = file.toString().endsWith(".gz") ? new GZIPInputStream(new ByteArrayInputStream(Files.readAllBytes(file))).readAllBytes() : Files.readAllBytes(file); - json.put(filenameBuilder.substring(1), Base64.getEncoder().encodeToString(fileData)); - } - return json; - } - void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { try { for (Path file : getMatchingFiles(earliestLogThreshold, latestLogThreshold)) { diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java index d578a745c9e..f0885451431 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogHandlerTest.java @@ -3,12 +3,11 @@ package com.yahoo.container.handler; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.time.Instant; import java.util.concurrent.Executor; @@ -17,7 +16,6 @@ import static org.mockito.Mockito.mock; public class LogHandlerTest { - @Test public void handleCorrectlyParsesQueryParameters() throws IOException { MockLogReader mockLogReader = new MockLogReader(); @@ -28,7 +26,7 @@ public class LogHandlerTest { HttpResponse response = logHandler.handle(HttpRequest.createTestRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.GET)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); response.render(bos); - String expectedResponse = "{\"logs\":{\"one\":\"newer_log\"}}"; + String expectedResponse = "newer log"; assertEquals(expectedResponse, bos.toString()); } @@ -37,7 +35,7 @@ public class LogHandlerTest { HttpResponse response = logHandler.handle(HttpRequest.createTestRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.GET)); ByteArrayOutputStream bos = new ByteArrayOutputStream(); response.render(bos); - String expectedResponse = "{\"logs\":{\"two\":\"older_log\"}}"; + String expectedResponse = "older log"; assertEquals(expectedResponse, bos.toString()); } @@ -49,12 +47,14 @@ public class LogHandlerTest { } @Override - protected JSONObject readLogs(Instant earliestLogThreshold, Instant latestLogThreshold) throws JSONException { - if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { - return new JSONObject("{\"one\":\"newer_log\"}"); - } else { - return new JSONObject("{\"two\":\"older_log\"}"); - } + protected void writeLogs(OutputStream outputStream, Instant earliestLogThreshold, Instant latestLogThreshold) { + try { + if (latestLogThreshold.isAfter(Instant.ofEpochMilli(1000))) { + outputStream.write("newer log".getBytes()); + } else { + outputStream.write("older log".getBytes()); + } + } catch (Exception e) {} } } } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index 464d6f772eb..27300e5f3f5 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -2,7 +2,6 @@ package com.yahoo.container.handler; import com.yahoo.vespa.test.file.TestFileSystem; -import org.json.JSONObject; import org.junit.Before; import org.junit.Test; @@ -15,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.util.Scanner; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -37,47 +35,37 @@ public class LogReaderTest { Files.setLastModifiedTime( Files.write(logDirectory.resolve("subfolder/log2.log"), "This is another log file\n".getBytes()), FileTime.from(Instant.ofEpochMilli(234))); - } - @Test - public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ - LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\",\"log1.log.gz\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGUK\"}"; - String actual = json.toString(); - assertEquals(expected, actual); } @Test public void testThatLogsOutsideRangeAreExcluded() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); - JSONObject json = logReader.readLogs(Instant.MAX, Instant.MIN); - String expected = "{}"; - String actual = json.toString(); + logReader.writeLogs(baos, Instant.ofEpochMilli(235), Instant.ofEpochMilli(300)); + String expected = ""; + String actual = decompress(baos.toByteArray()); assertEquals(expected, actual); } @Test public void testThatLogsNotMatchingRegexAreExcluded() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*2\\.log")); - JSONObject json = logReader.readLogs(Instant.ofEpochMilli(21), Instant.now()); - String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxlCg==\"}"; - String actual = json.toString(); + logReader.writeLogs(baos, Instant.ofEpochMilli(21), Instant.now()); + String expected = "This is another log file\n"; + String actual = decompress(baos.toByteArray()); assertEquals(expected, actual); } @Test public void testZippedStreaming() throws IOException { - ByteArrayOutputStream zippedBaos = new ByteArrayOutputStream(); LogReader logReader = new LogReader(logDirectory, Pattern.compile(".*")); logReader.writeLogs(zippedBaos, Instant.ofEpochMilli(21), Instant.now()); - GZIPInputStream unzippedIs = new GZIPInputStream(new ByteArrayInputStream(zippedBaos.toByteArray())); - - Scanner s = new Scanner(unzippedIs).useDelimiter("\\A"); - String actual = s.hasNext() ? s.next() : ""; String expected = "This is one log file\nThis is another log file\n"; + String actual = decompress(zippedBaos.toByteArray()); assertEquals(expected, actual); } @@ -89,4 +77,10 @@ public class LogReaderTest { return baos.toByteArray(); } + private String decompress(byte[] input) throws IOException { + if (input.length == 0) return ""; + byte[] decompressed = new GZIPInputStream(new ByteArrayInputStream(input)).readAllBytes(); + return new String(decompressed); + } + } diff --git a/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java b/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java index 4245f51ace8..565a4c483c3 100644 --- a/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java +++ b/container-search/src/main/java/com/yahoo/fs4/MapEncoder.java @@ -20,8 +20,13 @@ public class MapEncoder { // TODO: Time to refactor - private static final String TYPE_SUFFIX = ".type"; - private static final String TENSOR_TYPE = "tensor"; + private static byte [] getUtf8(Object value) { + if (value instanceof Tensor) { + return TypedBinaryFormat.encode((Tensor)value); + } else { + return Utf8.toBytes(value.toString()); + } + } /** * Encodes a single value as a complete binary map. @@ -39,7 +44,7 @@ public class MapEncoder { utf8 = Utf8.toBytes(key); buffer.putInt(utf8.length); buffer.put(utf8); - utf8 = Utf8.toBytes(value.toString()); + utf8 = getUtf8(value); buffer.putInt(utf8.length); buffer.put(utf8); @@ -64,7 +69,12 @@ public class MapEncoder { utf8 = Utf8.toBytes(key); buffer.putInt(utf8.length); buffer.put(utf8); - utf8 = Utf8.toBytes(property.getValue() != null ? property.getValue().toString() : ""); + Object value = property.getValue(); + if (value == null) { + utf8 = Utf8.toBytes(""); + } else { + utf8 = getUtf8(value); + } buffer.putInt(utf8.length); buffer.put(utf8); } @@ -78,53 +88,21 @@ public class MapEncoder { * * Returns the number of maps encoded - 0 or 1 */ - public static int encodeStringMultiMap(String mapName, Map<String,List<String>> map, ByteBuffer buffer) { - if (map.isEmpty()) return 0; - - byte [] utf8 = Utf8.toBytes(mapName); - buffer.putInt(utf8.length); - buffer.put(utf8); - buffer.putInt(countStringEntries(map)); - for (Map.Entry<String, List<String>> property : map.entrySet()) { - String key = property.getKey(); - for (Object value : property.getValue()) { - utf8 = Utf8.toBytes(key); - buffer.putInt(utf8.length); - buffer.put(utf8); - utf8 = Utf8.toBytes(value.toString()); - buffer.putInt(utf8.length); - buffer.put(utf8); - } - } - - return 1; - } - /** - * Encodes a multi-map as binary. - * Does nothing if the value is null. - * - * Returns the number of maps encoded - 0 or 1 - */ - public static int encodeObjectMultiMap(String mapName, Map<String,List<Object>> map, ByteBuffer buffer) { + public static <T> int encodeMultiMap(String mapName, Map<String,List<T>> map, ByteBuffer buffer) { if (map.isEmpty()) return 0; byte[] utf8 = Utf8.toBytes(mapName); buffer.putInt(utf8.length); buffer.put(utf8); - addTensorTypeInfo(map); - buffer.putInt(countObjectEntries(map)); - for (Map.Entry<String, List<Object>> property : map.entrySet()) { + buffer.putInt(countEntries(map)); + for (Map.Entry<String, List<T>> property : map.entrySet()) { String key = property.getKey(); for (Object value : property.getValue()) { utf8 = Utf8.toBytes(key); buffer.putInt(utf8.length); buffer.put(utf8); - if (value instanceof Tensor) { - utf8 = TypedBinaryFormat.encode((Tensor)value); - } else { - utf8 = Utf8.toBytes(value.toString()); - } + utf8 = getUtf8(value); buffer.putInt(utf8.length); buffer.put(utf8); } @@ -133,32 +111,9 @@ public class MapEncoder { return 1; } - private static void addTensorTypeInfo(Map<String, List<Object>> map) { - Map<String, Tensor> tensorsToTag = new HashMap<>(); - for (Map.Entry<String, List<Object>> entry : map.entrySet()) { - for (Object value : entry.getValue()) { - if (value instanceof Tensor) { - tensorsToTag.put(entry.getKey(), (Tensor)value); - } - } - } - for (Map.Entry<String, Tensor> entry : tensorsToTag.entrySet()) { - // Ensure that we only have a single tensor associated with each key - map.put(entry.getKey(), Arrays.asList(entry.getValue())); - map.put(entry.getKey() + TYPE_SUFFIX, Arrays.asList(TENSOR_TYPE)); - } - } - - private static int countStringEntries(Map<String, List<String>> value) { - int entries = 0; - for (Map.Entry<String, List<String>> property : value.entrySet()) - entries += property.getValue().size(); - return entries; - } - - private static int countObjectEntries(Map<String, List<Object>> value) { + private static <T> int countEntries(Map<String, List<T>> value) { int entries = 0; - for (Map.Entry<String, List<Object>> property : value.entrySet()) + for (Map.Entry<String, List<T>> property : value.entrySet()) entries += property.getValue().size(); return entries; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 5e994dac5d6..499cacd89c5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -13,7 +13,8 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; /** * Parser for queries of type all. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @author bratseth */ public class AllParser extends SimpleParser { @@ -32,37 +33,37 @@ public class AllParser extends SimpleParser { protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and and'ed items, then combine. - AndItem and=null; - NotItem not=null; // Store negatives here as we go + AndItem and = null; + NotItem not = null; // Store negatives here as we go Item current; // Find all items do { - current=negativeItem(); - if (current!=null) { - not=addNot(current,not); + current = negativeItem(); + if (current != null) { + not = addNot(current, not); continue; } - current=positiveItem(); - if (current==null) + current = positiveItem(); + if (current == null) current = indexableItem(); if (current == null) current = compositeItem(); - if (current!=null) - and=addAnd(current,and); + if (current != null) + and = addAnd(current, and); if (current == null) tokens.skip(); } while (tokens.hasNext()); // Combine the items - Item topLevel=and; + Item topLevel = and; - if (not!=null && topLevel!=null) { + if (not != null && topLevel != null) { not.setPositiveItem(topLevel); - topLevel=not; + topLevel = not; } return simplifyUnnecessaryComposites(topLevel); @@ -78,23 +79,23 @@ public class AllParser extends SimpleParser { return root.getRoot() instanceof NullItem ? null : root.getRoot(); } - protected AndItem addAnd(Item item,AndItem and) { - if (and==null) - and=new AndItem(); + protected AndItem addAnd(Item item, AndItem and) { + if (and == null) + and = new AndItem(); and.addItem(item); return and; } protected OrItem addOr(Item item,OrItem or) { - if (or==null) - or=new OrItem(); + if (or == null) + or = new OrItem(); or.addItem(item); return or; } protected NotItem addNot(Item item,NotItem not) { - if (not==null) - not=new NotItem(); + if (not == null) + not = new NotItem(); not.addNegativeItem(item); return not; } @@ -103,8 +104,7 @@ public class AllParser extends SimpleParser { int position = tokens.getPosition(); Item item = null; try { - if (!tokens.skipMultiple(MINUS)) return null; - + if ( ! tokens.skip(MINUS)) return null; if (tokens.currentIsNoIgnore(SPACE)) return null; item = indexableItem(); @@ -122,8 +122,15 @@ public class AllParser extends SimpleParser { } } } - if (item!=null) + if (item != null) item.setProtected(true); + + // Heuristic overdrive engaged! + // Interpret -N as a positive item matching a negative number (by backtracking out of this) + // but interpret --N as a negative item matching a negative number + if ( item instanceof IntItem && ! ((IntItem)item).getNumber().startsWith(("-"))) + item = null; + return item; } finally { if (item == null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 291beb40b4c..9ddfea6dffb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -10,10 +10,10 @@ import static com.yahoo.prelude.query.parser.Token.Kind.PLUS; import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; /** - * Base class for parsers of the "simple" query languages (query types - * ANY and ALL). + * Base class for parsers of the "simple" query languages (query types ANY and ALL). * * @author Steinar Knutsen + * @author bratseth */ abstract class SimpleParser extends StructuredParser { @@ -25,7 +25,6 @@ abstract class SimpleParser extends StructuredParser { return anyItems(false); // Nesteds are any even if all on top level } - protected abstract Item negativeItem(); /** @@ -163,11 +162,7 @@ abstract class SimpleParser extends StructuredParser { return false; } - - /** - * Removes and returns the first <i>not</i> found in the composite, - * or returns null if there's none - */ + /** Removes and returns the first <i>not</i> found in the composite, or returns null if there's none */ private NotItem removeNot(CompositeItem composite) { for (int i = 0; i < composite.getItemCount(); i++) { if (composite.getItem(i) instanceof NotItem) { @@ -184,7 +179,7 @@ abstract class SimpleParser extends StructuredParser { Item item = null; try { - if (!tokens.skipMultiple(PLUS)) { + if ( ! tokens.skipMultiple(PLUS)) { return null; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index ec1f79828c1..8ecd4d8f81c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -60,7 +60,7 @@ abstract class StructuredParser extends AbstractParser { String indexName = indexPrefix(); setSubmodeFromIndex(indexName, indexFacts); - item = number(indexName != null); + item = number(); if (item == null) { item = phrase(); @@ -147,39 +147,36 @@ abstract class StructuredParser extends AbstractParser { List<Token> firstWord = new ArrayList<>(); List<Token> secondWord = new ArrayList<>(); - tokens.skip(LSQUAREBRACKET); // For test 93 and 60 + tokens.skip(LSQUAREBRACKET); - if (!tokens.currentIs(WORD) && !tokens.currentIs(NUMBER) - && !tokens.currentIs(UNDERSCORE)) { + if ( ! tokens.currentIs(WORD) && ! tokens.currentIs(NUMBER) && ! tokens.currentIs(UNDERSCORE)) { return null; } firstWord.add(tokens.next()); while (tokens.currentIsNoIgnore(UNDERSCORE) - || tokens.currentIsNoIgnore(WORD) - || tokens.currentIsNoIgnore(NUMBER)) { + || tokens.currentIsNoIgnore(WORD) + || tokens.currentIsNoIgnore(NUMBER)) { firstWord.add(tokens.next()); } if (tokens.currentIsNoIgnore(DOT)) { tokens.skip(); - if (tokens.currentIsNoIgnore(WORD) - || tokens.currentIsNoIgnore(NUMBER)) { + if (tokens.currentIsNoIgnore(WORD) || tokens.currentIsNoIgnore(NUMBER)) { secondWord.add(tokens.next()); } else { return null; } while (tokens.currentIsNoIgnore(UNDERSCORE) - || tokens.currentIsNoIgnore(WORD) - || tokens.currentIsNoIgnore(NUMBER)) { + || tokens.currentIsNoIgnore(WORD) + || tokens.currentIsNoIgnore(NUMBER)) { secondWord.add(tokens.next()); } } - if (!tokens.skipNoIgnore(COLON)) { + if ( ! tokens.skipNoIgnore(COLON)) return null; - } if (secondWord.size() == 0) { item = concatenate(firstWord); @@ -195,8 +192,7 @@ abstract class StructuredParser extends AbstractParser { return null; } else { if (nothingAhead(false)) { - // correct index syntax, correct name, but followed - // by noise. Let's skip this. + // correct index syntax, correct name, but followed by noise. Let's skip this. nothingAhead(true); position = tokens.getPosition(); item = indexPrefix(); @@ -253,11 +249,11 @@ abstract class StructuredParser extends AbstractParser { private boolean endOfNumber() { return tokens.currentIsNoIgnore(SPACE) - || tokens.currentIsNoIgnore(RSQUAREBRACKET) - || tokens.currentIsNoIgnore(SEMICOLON) - || tokens.currentIsNoIgnore(RBRACE) - || tokens.currentIsNoIgnore(EOF) - || tokens.currentIsNoIgnore(EXCLAMATION); + || tokens.currentIsNoIgnore(RSQUAREBRACKET) + || tokens.currentIsNoIgnore(SEMICOLON) + || tokens.currentIsNoIgnore(RBRACE) + || tokens.currentIsNoIgnore(EOF) + || tokens.currentIsNoIgnore(EXCLAMATION); } private String decimalPart() { @@ -277,19 +273,19 @@ abstract class StructuredParser extends AbstractParser { } } - private IntItem number(boolean hasIndex) { + private IntItem number() { int position = tokens.getPosition(); IntItem item = null; try { - if (item == null) { - item = numberRange(); - } + item = numberRange(); - tokens.skip(LSQUAREBRACKET); // For test 93 and 60 + tokens.skip(LSQUAREBRACKET); + if (item == null) + tokens.skipNoIgnore(SPACE); // TODO: Better definition of start and end of numeric items - if (item == null && hasIndex && tokens.currentIsNoIgnore(MINUS) && (tokens.currentNoIgnore(1).kind == NUMBER)) { + if (item == null && tokens.currentIsNoIgnore(MINUS) && (tokens.currentNoIgnore(1).kind == NUMBER)) { tokens.skipNoIgnore(); Token t = tokens.next(); item = new IntItem("-" + t.toString() + decimalPart(), true); @@ -307,7 +303,7 @@ abstract class StructuredParser extends AbstractParser { if (item == null) { item = numberGreater(); } - if (item != null && !endOfNumber()) { + if (item != null && ! endOfNumber()) { item = null; } return item; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java index 42cef67f189..fbaf1675ff1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java @@ -1,10 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; - import java.util.List; - /** * An iterator-like view of a list of tokens, but typed, random-accessible * and with more convenience methods @@ -183,8 +181,7 @@ final class TokenPosition { /** * Skips one or zero items of the given kind. * - * @return true if one item was skipped, false if none was, - * or if there are no more tokens + * @return true if one item was skipped, false if none was, or if there are no more tokens */ public boolean skip(Token.Kind kind) { Token current = current(); @@ -198,20 +195,16 @@ final class TokenPosition { } /** - * Skips one or zero items of the given kind, without ignoring - * spaces + * Skips one or zero items of the given kind, without ignoring spaces * - * @return true if one item was skipped, false if none was, - * or if there are no more tokens + * @return true if one item was skipped, false if none was or if there are no more tokens */ public boolean skipNoIgnore(Token.Kind kind) { Token current = currentNoIgnore(); - if (current == null || current.kind != kind) { - return false; - } + if (current == null || current.kind != kind) return false; - skip(); + skipNoIgnore(); return true; } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index b97ee87f650..a5007c9cc33 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -1055,7 +1055,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // TODO: Push down if (presentation.getHighlight() != null) { - mapCount += MapEncoder.encodeStringMultiMap(Highlight.HIGHLIGHTTERMS, presentation.getHighlight().getHighlightTerms(), buffer); + mapCount += MapEncoder.encodeMultiMap(Highlight.HIGHLIGHTTERMS, presentation.getHighlight().getHighlightTerms(), buffer); } // TODO: Push down diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java index 32c386f0e32..b3d79f65df4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java @@ -15,8 +15,7 @@ public interface Parser { * {@link QueryTree}. If parsing fails without an exception, the contained * root will be an instance of {@link com.yahoo.prelude.query.NullItem}. * - * @param query - * the Parsable to parse + * @param query the Parsable to parse * @return the parsed QueryTree, never null */ QueryTree parse(Parsable query); diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java index 4158b0e7476..37a54a82c43 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java @@ -76,7 +76,7 @@ public class RankProperties implements Cloneable { /** Encodes this in a binary internal representation and returns the number of property maps encoded (0 or 1) */ public int encode(ByteBuffer buffer, boolean encodeQueryData) { if (encodeQueryData) { - return MapEncoder.encodeObjectMultiMap("rank", properties, buffer); + return MapEncoder.encodeMultiMap("rank", properties, buffer); } else { List<Object> sessionId = properties.get(GetDocSumsPacket.sessionIdKey); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index af095fefc1c..3eac1d88784 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -712,7 +712,7 @@ public class YqlParser implements Parser { .setLanguage(language) .setDefaultIndexName(defaultIndex)).getRoot(); // the null check should be unnecessary, but is there to avoid having to suppress null warnings - if ( !allowNullItem && (item == null || item instanceof NullItem)) + if ( ! allowNullItem && (item == null || item instanceof NullItem)) throw new IllegalArgumentException("Parsing '" + wordData + "' only resulted in NullItem."); if (language != Language.ENGLISH) // mark the language used, unless it's the default diff --git a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java index 69ca646dbd5..e8c16e572ae 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java @@ -63,11 +63,10 @@ public class RankFeaturesTestCase { assertEquals(entries.size(), properties.asMap().size()); Map<String, Object> decodedProperties = decode(type, encode(properties)); - assertEquals(entries.size() * 2, properties.asMap().size()); // tensor type info has been added - assertEquals(entries.size() * 2, decodedProperties.size()); + assertEquals(entries.size(), properties.asMap().size()); + assertEquals(entries.size(), decodedProperties.size()); for (Entry entry : entries) { assertEquals(entry.tensor, decodedProperties.get(entry.normalizedKey)); - assertEquals("tensor", decodedProperties.get(entry.normalizedKey + ".type")); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 497f05e8341..04b1d526c67 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -145,7 +145,8 @@ public class FastSearcherTestCase { doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", - "Error response from rpc node connection to host1:0: Connection error", error.getDetailedMessage()); + "Error response from rpc node connection to hostX:0: Connection error", + error.getDetailedMessage().replaceAll("host[12]", "hostX")); } { // direct.summaries due to no summary features diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 12f9ef2b18f..73f2ae7eb87 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -1957,7 +1957,12 @@ public class ParseTestCase { @Test public void testNumbersAndNot() { - tester.assertParsed("+a -12", "a -12", Query.Type.ALL); + tester.assertParsed("AND a -12", "a -12", Query.Type.ALL); + } + + @Test + public void testNumbersAndDoubleNot() { + tester.assertParsed("+a --12", "a --12", Query.Type.ALL); } @Test @@ -1967,7 +1972,7 @@ public class ParseTestCase { @Test public void testSingleNegativeNumberLikeTerm() { - tester.assertParsed(null, "-12", Query.Type.ALL); + tester.assertParsed("-12", "-12", Query.Type.ALL); } @Test @@ -2004,7 +2009,12 @@ public class ParseTestCase { @Test public void testDecimalNumbersAndNot() { - tester.assertParsed("+a -12.2", "a -12.2", Query.Type.ALL); + tester.assertParsed("AND a -12.2", "a -12.2", Query.Type.ALL); + } + + @Test + public void testDecimalNumbersAndDoubleNot() { + tester.assertParsed("+a --12.2", "a --12.2", Query.Type.ALL); } @Test @@ -2014,7 +2024,7 @@ public class ParseTestCase { @Test public void testSingleNegativeDecimalNumberLikeTerm() { - tester.assertParsed(null, "-12.2", Query.Type.ALL); + tester.assertParsed("-12.2", "-12.2", Query.Type.ALL); } @Test @@ -2321,12 +2331,12 @@ public class ParseTestCase { @Test public void testSingleNegativeNumberLikeTermWeb() { - tester.assertParsed(null, "-12", Query.Type.WEB); + tester.assertParsed("-12", "-12", Query.Type.WEB); } @Test public void testSingleNegativeDecimalNumberLikeTermWeb() { - tester.assertParsed(null, "-12.2", Query.Type.WEB); + tester.assertParsed("-12.2", "-12.2", Query.Type.WEB); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index b5c4166e4de..6173d710434 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -118,6 +118,17 @@ public class UserInputTestCase { } @Test + public void testNegativeNumberComparison() { + URIBuilder builder = searchUri(); + builder.setParameter("myinput", "-5"); + builder.setParameter("yql", + "select * from ecitem where rank(([{\"defaultIndex\":\"myfield\"}](userInput(@myinput))));"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from ecitem where rank(myfield = (-5));", query.yqlRepresentation()); + assertEquals("RANK myfield:-5", query.getModel().getQueryTree().getRoot().toString()); + } + + @Test public void testAnnotatedUserInputUnrankedTerms() { URIBuilder builder = searchUri(); builder.setParameter("yql", diff --git a/controller-api/CMakeLists.txt b/controller-api/CMakeLists.txt new file mode 100644 index 00000000000..ea02292d1eb --- /dev/null +++ b/controller-api/CMakeLists.txt @@ -0,0 +1 @@ +install_configserver_component(controller-api) diff --git a/functions.cmake b/functions.cmake index bfb4c945c16..2b460c8c5ea 100644 --- a/functions.cmake +++ b/functions.cmake @@ -566,6 +566,22 @@ function(install_symlink TARGET LINK) install_absolute_symlink(${CMAKE_INSTALL_PREFIX}/${TARGET} ${LINK}) endfunction(install_symlink) +function(install_configserver_component NAME) + cmake_parse_arguments( + PARAM + "" + "CLASSIFIER" + "" + ${ARGN} + ) + if(NOT PARAM_CLASSIFIER) + SET(PARAM_CLASSIFIER "jar-with-dependencies") + endif() + install(FILES "target/${NAME}-${PARAM_CLASSIFIER}.jar" DESTINATION lib/jars/) + install(DIRECTORY DESTINATION conf/configserver-app/components) + install_symlink(lib/jars/${NAME}-${PARAM_CLASSIFIER}.jar conf/configserver-app/components/${NAME}.jar) +endfunction() + function(add_extra_projects) if(EXTRA_PROJECTS) foreach(PROJECT ${EXTRA_PROJECTS}) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java index b6591ef4825..574be20e910 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java @@ -33,6 +33,9 @@ public class BindingRepository<T> implements Iterable<Map.Entry<UriPattern, T>> * @throws IllegalArgumentException If the URI pattern string could not be parsed. */ public void bind(String uriPattern, T target) { + if (uriPattern.startsWith("https://")) { + log.warning(() -> "Bindings with 'https' scheme are deprecated. Use 'http' to match both 'http' and 'https' in URIs."); + } put(new UriPattern(uriPattern), target); } diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index f46a2d9d2e3..2bc3ae5cfeb 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -8,12 +8,10 @@ <handler id="com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler" bundle="node-repository"> <binding>http://*/nodes/v2/*</binding> - <binding>https://*/nodes/v2/*</binding> </handler> <handler id="com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler" bundle="node-repository"> <binding>http://*/loadbalancers/v1/*</binding> - <binding>https://*/loadbalancers/v1/*</binding> </handler> <preprocess:include file="node-flavors.xml" required="false" /> diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 0da0fd5ce66..350cdad791c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -6,14 +6,14 @@ namespace storage { FileStorHandler::FileStorHandler(MessageSender& sender, FileStorMetrics& metrics, const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _impl(new FileStorHandlerImpl(1, sender, metrics, partitions, compReg)) + : _impl(new FileStorHandlerImpl(1, 1, sender, metrics, partitions, compReg)) { } -FileStorHandler::FileStorHandler(uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, +FileStorHandler::FileStorHandler(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) - : _impl(new FileStorHandlerImpl(numStripes, sender, metrics, partitions, compReg)) + : _impl(new FileStorHandlerImpl(numThreads, numStripes, sender, metrics, partitions, compReg)) { } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index 8f0b060d4d6..ab3d03a5e9a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -71,7 +71,7 @@ public: CLOSED }; - FileStorHandler(uint32_t numStripes, MessageSender&, FileStorMetrics&, + FileStorHandler(uint32_t numThreads, uint32_t numStripes, MessageSender&, FileStorMetrics&, const spi::PartitionStateList&, ServiceLayerComponentRegister&); FileStorHandler(MessageSender&, FileStorMetrics&, const spi::PartitionStateList&, ServiceLayerComponentRegister&); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 78498be2510..a56104ff717 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -22,7 +22,23 @@ using document::BucketSpace; namespace storage { -FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, +namespace { + +uint32_t merge_soft_limit_from_thread_count(uint32_t num_threads) noexcept { + // Rationale: to avoid starving client ops we want to ensure that not all persistence + // threads can be blocked by processing merges all at the same time. We therefore allocate + // half of the threads to non-merge operations. + // This a _soft_ limit since the current operation locking design means there is a small + // window of time between when the limit is checked and when its updated. There are no + // correctness violations as a consequence of this, but non-merge liveness may be impacted. + // There must always be at least 1 thread that can process merges, or the system would stall. + return std::max(1u, num_threads / 2); +} + +} + +FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, + FileStorMetrics& metrics, [[maybe_unused]] const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg) : _component(compReg, "filestorhandlerimpl"), @@ -30,6 +46,8 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numStripes, MessageSender& sen _messageSender(sender), _bucketIdFactory(_component.getBucketIdFactory()), _getNextMessageTimeout(100), + _activeMergesSoftLimit(merge_soft_limit_from_thread_count(numThreads)), + _activeMerges(0), _paused(false) { _diskInfo.reserve(_component.getDiskCount()); @@ -926,7 +944,7 @@ FileStorHandlerImpl::Stripe::getNextMessage(uint32_t timeout, Disk & disk) PriorityIdx& idx(bmi::get<1>(_queue)); PriorityIdx::iterator iter(idx.begin()), end(idx.end()); - while (iter != end && isLocked(guard, iter->_bucket, iter->_command->lockingRequirements())) { + while (iter != end && operationIsInhibited(guard, iter->_bucket, *iter->_command)) { iter++; } if (iter != end) { @@ -1105,6 +1123,10 @@ void FileStorHandlerImpl::Stripe::release(const document::Bucket & bucket, if (reqOfReleasedLock == api::LockingRequirements::Exclusive) { assert(entry._exclusiveLock); assert(entry._exclusiveLock->msgId == lockMsgId); + if (entry._exclusiveLock->msgType == api::MessageType::MERGEBUCKET_ID) { + auto before = _owner._activeMerges.fetch_sub(1, std::memory_order_relaxed); + assert(before > 0); + } entry._exclusiveLock.reset(); } else { assert(!entry._exclusiveLock); @@ -1125,6 +1147,9 @@ void FileStorHandlerImpl::Stripe::lock(const vespalib::MonitorGuard &, const doc assert(!entry._exclusiveLock); if (lockReq == api::LockingRequirements::Exclusive) { assert(entry._sharedLocks.empty()); + if (lockEntry.msgType == api::MessageType::MERGEBUCKET_ID) { + _owner._activeMerges.fetch_add(1, std::memory_order_relaxed); + } entry._exclusiveLock = lockEntry; } else { // TODO use a hash set with a custom comparator/hasher instead...? @@ -1154,6 +1179,18 @@ FileStorHandlerImpl::Stripe::isLocked(const vespalib::MonitorGuard &, const docu && !iter->second._sharedLocks.empty()); } +bool +FileStorHandlerImpl::Stripe::operationIsInhibited(const vespalib::MonitorGuard& guard, const document::Bucket& bucket, + const api::StorageMessage& msg) const noexcept +{ + if ((msg.getType() == api::MessageType::MERGEBUCKET) + && (_owner._activeMerges.load(std::memory_order_relaxed) > _owner._activeMergesSoftLimit)) + { + return true; + } + return isLocked(guard, bucket, msg.lockingRequirements()); +} + uint32_t FileStorHandlerImpl::Disk::getQueueSize() const noexcept { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 390b7284b85..d46aa635354 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -115,6 +115,9 @@ public: void release(const document::Bucket & bucket, api::LockingRequirements reqOfReleasedLock, api::StorageMessage::Id lockMsgId); + // Subsumes isLocked + bool operationIsInhibited(const vespalib::MonitorGuard&, const document::Bucket&, + const api::StorageMessage&) const noexcept; bool isLocked(const vespalib::MonitorGuard &, const document::Bucket&, api::LockingRequirements lockReq) const noexcept; @@ -230,7 +233,7 @@ public: api::LockingRequirements _lockReq; }; - FileStorHandlerImpl(uint32_t numStripes, MessageSender&, FileStorMetrics&, + FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender&, FileStorMetrics&, const spi::PartitionStateList&, ServiceLayerComponentRegister&); ~FileStorHandlerImpl(); @@ -293,6 +296,8 @@ private: uint32_t _getNextMessageTimeout; + uint32_t _activeMergesSoftLimit; + mutable std::atomic<uint32_t> _activeMerges; vespalib::Monitor _pauseMonitor; std::atomic<bool> _paused; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 76e04852178..baa6523cbb2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -108,7 +108,7 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC size_t numStripes = std::min(2ul, numThreads); _metrics->initDiskMetrics(_disks.size(), _component.getLoadTypes()->getMetricLoadTypes(), numStripes, numThreads); - _filestorHandler.reset(new FileStorHandler(numStripes, *this, *_metrics, _partitions, _compReg)); + _filestorHandler.reset(new FileStorHandler(numThreads, numStripes, *this, *_metrics, _partitions, _compReg)); for (uint32_t i=0; i<_component.getDiskCount(); ++i) { if (_partitions[i].isUp()) { LOG(spam, "Setting up disk %u", i); |