diff options
298 files changed, 5211 insertions, 6039 deletions
diff --git a/application/src/test/java/com/yahoo/application/container/MockClient.java b/application/src/test/java/com/yahoo/application/container/MockClient.java index 1f885cfb737..eee12478592 100644 --- a/application/src/test/java/com/yahoo/application/container/MockClient.java +++ b/application/src/test/java/com/yahoo/application/container/MockClient.java @@ -1,43 +1,43 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container;
-
-import com.yahoo.jdisc.Request;
-import com.yahoo.jdisc.Response;
-import com.yahoo.jdisc.handler.CompletionHandler;
-import com.yahoo.jdisc.handler.ContentChannel;
-import com.yahoo.jdisc.handler.ResponseHandler;
-import com.yahoo.jdisc.service.AbstractClientProvider;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- *
- * @author Christian Andersen
- */
-public class MockClient extends AbstractClientProvider {
- private final AtomicInteger counter = new AtomicInteger();
-
- @Override
- public ContentChannel handleRequest(Request request, ResponseHandler handler) {
- counter.incrementAndGet();
- final ContentChannel responseContentChannel = handler.handleResponse(new Response(200));
- responseContentChannel.close(NOOP_COMPLETION_HANDLER);
- return null;
- }
-
- public int getCounter() {
- return counter.get();
- }
-
- private static final CompletionHandler NOOP_COMPLETION_HANDLER = new CompletionHandler() {
- @Override
- public void completed() {
- // Ignored
- }
-
- @Override
- public void failed(Throwable t) {
- // Ignored
- }
- };
-}
+package com.yahoo.application.container; + +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.CompletionHandler; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.service.AbstractClientProvider; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * + * @author Christian Andersen + */ +public class MockClient extends AbstractClientProvider { + private final AtomicInteger counter = new AtomicInteger(); + + @Override + public ContentChannel handleRequest(Request request, ResponseHandler handler) { + counter.incrementAndGet(); + final ContentChannel responseContentChannel = handler.handleResponse(new Response(200)); + responseContentChannel.close(NOOP_COMPLETION_HANDLER); + return null; + } + + public int getCounter() { + return counter.get(); + } + + private static final CompletionHandler NOOP_COMPLETION_HANDLER = new CompletionHandler() { + @Override + public void completed() { + // Ignored + } + + @Override + public void failed(Throwable t) { + // Ignored + } + }; +} diff --git a/application/src/test/java/com/yahoo/application/container/MockServer.java b/application/src/test/java/com/yahoo/application/container/MockServer.java index bc79671c13a..932f95647e7 100644 --- a/application/src/test/java/com/yahoo/application/container/MockServer.java +++ b/application/src/test/java/com/yahoo/application/container/MockServer.java @@ -1,31 +1,31 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container;
-
-import com.yahoo.jdisc.service.AbstractServerProvider;
-import com.yahoo.jdisc.service.CurrentContainer;
-
-/**
- *
- * @author Christian Andersen
- */
-public class MockServer extends AbstractServerProvider {
- private boolean started = false;
-
- public MockServer(CurrentContainer container) {
- super(container);
- }
-
- @Override
- public void start() {
- started = true;
- }
-
- @Override
- public void close() {
-
- }
-
- public boolean isStarted() {
- return started;
- }
-}
+package com.yahoo.application.container; + +import com.yahoo.jdisc.service.AbstractServerProvider; +import com.yahoo.jdisc.service.CurrentContainer; + +/** + * + * @author Christian Andersen + */ +public class MockServer extends AbstractServerProvider { + private boolean started = false; + + public MockServer(CurrentContainer container) { + super(container); + } + + @Override + public void start() { + started = true; + } + + @Override + public void close() { + + } + + public boolean isStarted() { + return started; + } +} diff --git a/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java b/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java index 3c11cbdad8a..41266c46ce7 100644 --- a/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java +++ b/application/src/test/java/com/yahoo/application/container/docprocs/MockDispatchDocproc.java @@ -1,65 +1,65 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.application.container.docprocs;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.yahoo.docproc.DocumentProcessor;
-import com.yahoo.docproc.Processing;
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentOperation;
-import com.yahoo.document.DocumentPut;
-import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
-import com.yahoo.jdisc.Request;
-import com.yahoo.jdisc.Response;
-import com.yahoo.jdisc.handler.RequestDispatch;
-import com.yahoo.jdisc.service.CurrentContainer;
-import com.yahoo.messagebus.jdisc.MbusRequest;
-import com.yahoo.messagebus.routing.Route;
-
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-/**
- * @author Christian Andersen
- */
-public class MockDispatchDocproc extends DocumentProcessor {
- private final Route route;
- private final URI uri;
- private final CurrentContainer currentContainer;
- private final List<Response> responses = new ArrayList<>();
-
- public MockDispatchDocproc(CurrentContainer currentContainer) {
- this.route = Route.parse("default");
- this.uri = URI.create("mbus://remotehost/source");
- this.currentContainer = currentContainer;
- }
-
- @Override
- public Progress process(Processing processing) {
- for (DocumentOperation op : processing.getDocumentOperations()) {
- PutDocumentMessage message = new PutDocumentMessage((DocumentPut)op);
- ListenableFuture<Response> future = createRequest(message).dispatch();
- try {
- responses.add(future.get());
- } catch (ExecutionException | InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- return Progress.DONE;
- }
-
- private RequestDispatch createRequest(final DocumentMessage message) {
- return new RequestDispatch() {
- @Override
- protected Request newRequest() {
- return new MbusRequest(currentContainer, uri, message.setRoute(route)).setServerRequest(false);
- }
- };
- }
-
- public List<Response> getResponses() {
- return responses;
- }
-}
+package com.yahoo.application.container.docprocs; + +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.jdisc.Request; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.RequestDispatch; +import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.messagebus.jdisc.MbusRequest; +import com.yahoo.messagebus.routing.Route; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +/** + * @author Christian Andersen + */ +public class MockDispatchDocproc extends DocumentProcessor { + private final Route route; + private final URI uri; + private final CurrentContainer currentContainer; + private final List<Response> responses = new ArrayList<>(); + + public MockDispatchDocproc(CurrentContainer currentContainer) { + this.route = Route.parse("default"); + this.uri = URI.create("mbus://remotehost/source"); + this.currentContainer = currentContainer; + } + + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + PutDocumentMessage message = new PutDocumentMessage((DocumentPut)op); + ListenableFuture<Response> future = createRequest(message).dispatch(); + try { + responses.add(future.get()); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + return Progress.DONE; + } + + private RequestDispatch createRequest(final DocumentMessage message) { + return new RequestDispatch() { + @Override + protected Request newRequest() { + return new MbusRequest(currentContainer, uri, message.setRoute(route)).setServerRequest(false); + } + }; + } + + public List<Response> getResponses() { + return responses; + } +} diff --git a/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java b/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java index 106d68f6644..da1970e7522 100644 --- a/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java +++ b/application/src/test/java/com/yahoo/application/container/searchers/MockSearcher.java @@ -1,24 +1,24 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.application.container.searchers;
-
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.searchchain.Execution;
-
-/**
- *
- * @author Christian Andersen
- */
-public class MockSearcher extends Searcher {
-
- @Override
- public Result search(Query query, Execution execution) {
- HitGroup hits = new HitGroup();
- hits.add(new Hit("foo", query));
- return new Result(query, hits);
- }
-
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.searchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +/** + * + * @author Christian Andersen + */ +public class MockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + HitGroup hits = new HitGroup(); + hits.add(new Hit("foo", query)); + return new Result(query, hits); + } + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java index 7304d1e673a..fe4ac802bce 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRendrer.java @@ -301,4 +301,4 @@ public class VdsClusterHtmlRendrer { public Table createNewClusterHtmlTable(final String clusterName, final int slobrokGenerationCount) { return new Table(clusterName, slobrokGenerationCount); } -}
\ No newline at end of file +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java index 0d2f12dfdcd..4a1dd52a6bc 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java @@ -565,4 +565,4 @@ public class NodeStateChangeCheckerTest { return nodes; } -}
\ No newline at end of file +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java index fd93df4b2dc..0c61c3f62c8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java @@ -104,4 +104,4 @@ public class HostInfoTest { assertThat(storageNodeByIndex.get(5).getOpsLatenciesOrNull().getPut().getLatencyMsSum(), is(25000L)); assertThat(storageNodeByIndex.get(5).getOpsLatenciesOrNull().getPut().getCount(), is(7L)); } -}
\ No newline at end of file +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java index a6789ae22e5..8653c33c14e 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java @@ -102,4 +102,4 @@ public interface Waiter { } } -}
\ No newline at end of file +} diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 77842a693b7..67edf50851f 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -97,4 +97,4 @@ public class FilesApplicationPackageTest { assertTrue(app.getDeployment().isPresent()); assertThat(IOUtils.readAll(new FileReader(deployment)), is(IOUtils.readAll(app.getDeployment().get()))); } -}
\ No newline at end of file +} diff --git a/config-lib/src/main/java/com/yahoo/config/ConfigurationRuntimeException.java b/config-lib/src/main/java/com/yahoo/config/ConfigurationRuntimeException.java index 56020906499..91de5aafdd4 100644 --- a/config-lib/src/main/java/com/yahoo/config/ConfigurationRuntimeException.java +++ b/config-lib/src/main/java/com/yahoo/config/ConfigurationRuntimeException.java @@ -18,4 +18,4 @@ public class ConfigurationRuntimeException extends RuntimeException { super(cause); } -}
\ No newline at end of file +} diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java index eec2d3cd500..14c72ee8b14 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java @@ -3,6 +3,7 @@ package com.yahoo.config; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; import java.util.List; /** @@ -36,13 +37,24 @@ public class InnerNodeVector<NODE extends InnerNode> extends NodeVector<NODE> { /** * Creates a new Node by creating a new instance with the 0-argument constructor */ - // TODO: remove when the library uses reflection via builders + // TODO: remove, only used for NodeVector.setSize which only seems to be used by unit tests. @SuppressWarnings("unchecked") protected NODE createNew() { try { - Constructor<? extends InnerNode> ctor = defaultNode.getClass().getDeclaredConstructor(); - ctor.setAccessible(true); - return (NODE) ctor.newInstance(); + Class<? extends InnerNode> nodeClass = defaultNode.getClass(); + Class<?> builderClass = Arrays.stream(nodeClass.getClasses()) + .filter(klass -> klass.getSimpleName().equals("Builder")) + .findFirst() + .orElseThrow(() -> new ConfigurationRuntimeException("Could not find builder class for " + nodeClass.getName())); + + Constructor<?> builderCtor = builderClass.getConstructor(); + Object builderInstance = builderCtor.newInstance(); + if (! (builderInstance instanceof ConfigBuilder)) + throw new ConfigurationRuntimeException("Builder is not a ConfigBuilder, has class: " + builderInstance.getClass().getName()); + + Constructor<? extends InnerNode> nodeCtor = nodeClass.getDeclaredConstructor(builderClass, boolean.class); + nodeCtor.setAccessible(true); + return (NODE) nodeCtor.newInstance(builderInstance, false); } catch (InvocationTargetException | IllegalAccessException | InstantiationException | NoSuchMethodException e) { throw new ConfigurationRuntimeException(e); } diff --git a/config-lib/src/main/java/com/yahoo/config/NodeVector.java b/config-lib/src/main/java/com/yahoo/config/NodeVector.java index 8ce5689937e..5279db65b85 100644 --- a/config-lib/src/main/java/com/yahoo/config/NodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/NodeVector.java @@ -34,7 +34,7 @@ public abstract class NodeVector<NODE> implements java.util.List<NODE> { * * @param n the new size of this NodeVector */ - // TODO: remove when the library uses reflection via builders, and resizing won't be necessary + // TODO: remove, only used by unit tests public void setSize(int n) { while (size() > n) vector.remove(n); while (size() < n) vector.add(createNew()); diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index f99639420e2..86a141e5cab 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -181,4 +181,4 @@ public class ConfigInstanceEqualsTest { b(-2))); } -}
\ No newline at end of file +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java index 3e14485dd5a..5af4a8ea7ee 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java @@ -30,4 +30,4 @@ public interface HostProvisioner { */ List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); -}
\ No newline at end of file +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java index c58a116b196..e6d1731ddf8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.config.model.producer; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java index b10a8633a3a..7232a2c5299 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java @@ -71,4 +71,4 @@ class ConstantTensorTransformer extends ExpressionTransformer { return new ReferenceNode("constant", Arrays.asList(new NameNode(node.getName())), null); } -}
\ No newline at end of file +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java index 3df0c1f9953..70a51cfd7ba 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java @@ -37,4 +37,4 @@ public class UnprocessingSearchBuilder extends SearchBuilder { builder.build(); return builder.getSearch(); } -}
\ No newline at end of file +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java index 4e63729ea34..a44c2f45d6f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.searchdefinition.derived; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index bda79ee2356..dd6556f81d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -102,7 +102,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon Map<HostResource, List<SERVICE>> affinityMap = new HashMap<>(); for (SERVICE service : services) { if (!affinityMap.containsKey(service.getHostResource())) { - affinityMap.put(service.getHostResource(), new ArrayList<SERVICE>()); + affinityMap.put(service.getHostResource(), new ArrayList<>()); } int cpuSocket = affinityMap.get(service.getHostResource()).size(); affinityMap.get(service.getHostResource()).add(service); @@ -548,4 +548,9 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon public void setAffinity(Affinity affinity) { this.affinity = Optional.ofNullable(affinity); } + + @Override + public String toString() { + return getServiceName() + " on " + getHost().toString(); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index d54dc912da6..46dc287bb10 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -3,13 +3,11 @@ package com.yahoo.vespa.model; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -30,7 +28,7 @@ import java.util.stream.Collectors; public class HostResource implements Comparable<HostResource> { public final static int BASE_PORT = 19100; - public final static int MAX_PORTS = 799; + final static int MAX_PORTS = 799; private final Host host; // Map from "sentinel name" to service @@ -81,18 +79,6 @@ public class HostResource implements Comparable<HostResource> { 0; } - boolean isPortRangeAvailable(int start, int numPorts) { - int range = 0; - int port = start; - for (; port < BASE_PORT + MAX_PORTS && (range < numPorts); port++) { - if (portDB.containsKey(port)) { - return false; - } - range++; - } - return range == numPorts; - } - /** * Adds service and allocates resources for it. * @@ -179,7 +165,8 @@ public class HostResource implements Comparable<HostResource> { return port >= BASE_PORT && port < BASE_PORT + MAX_PORTS; } - private void portAlreadyReserved(AbstractService service, int port) { + + private void portAlreadyReserved(AbstractService service, int port) { AbstractService otherService = (AbstractService)portDB.get(port); int nextAvailablePort = nextAvailableBaseport(service.getPortCount()); if (nextAvailablePort == 0) { @@ -190,10 +177,9 @@ public class HostResource implements Comparable<HostResource> { : ""; throw new RuntimeException(service.getServiceName() + " cannot reserve port " + port + " on " + this + ": Already reserved for " + otherService.getServiceName() + - ". " + msg + "Next available port is: " + nextAvailablePort); + ". " + msg + "Next available port is: " + nextAvailablePort + " ports used: " + portDB); } - private void noMoreAvailablePorts() { throw new RuntimeException ("Too many ports are reserved in Vespa's port range (" + @@ -201,7 +187,6 @@ public class HostResource implements Comparable<HostResource> { ". Move one or more services to another host, or outside this port range."); } - /** * Returns the service with the given "sentinel name" on this Host, * or null if the name does not match any service. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java index a0b3cc7294b..e8f2bf42807 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java @@ -20,7 +20,7 @@ public class PortsMeta implements Serializable { private Integer rpcAdminOffset = null; /** Remember the rpc status port offset. */ private Integer rpcStatusOffset = null; - /** Remember the https admin port offset. */ + /** Remember the http admin port offset. */ private Integer httpAdminOffset = null; /** Remember the http status port offset. */ private Integer httpStatusOffset = null; @@ -64,7 +64,7 @@ public class PortsMeta implements Serializable { private PortsMeta register(int offset, String meta) { // Allocate new LinkedLists on each element up-to-and-including offset for (int i = ports.size(); i <= offset; i++) { - ports.add(i, new LinkedList<String>()); + ports.add(i, new LinkedList<>()); } ports.get(offset).addFirst(meta); @@ -88,6 +88,7 @@ public class PortsMeta implements Serializable { public int getNumPorts() { return ports.size(); } + /** * Get an iterator of the Strings registered at the specific point. * @param offset The relative offset to inquire about tags. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java index b2c2510ce22..cd06ae67776 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -42,16 +42,16 @@ public class Configserver extends AbstractService { * Returns the desired base port for this service. */ public int getWantedPort() { - try { - // TODO: Provide configserver port as argument when creating this service instead - Process process = new ProcessBuilder(getDefaults().underVespaHome("bin/vespa-print-default"), "configserver_rpc_port").start(); - InputStream in = process.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - return Integer.parseInt(reader.readLine().trim()); - } catch (Exception exception) { - log.log(LogLevel.DEBUG, "Error reading port from script, using " + defaultPort); - return defaultPort; - } + try { + // TODO: Provide configserver port as argument when creating this service instead + Process process = new ProcessBuilder(getDefaults().underVespaHome("bin/vespa-print-default"), "configserver_rpc_port").start(); + InputStream in = process.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + return Integer.parseInt(reader.readLine().trim()); + } catch (Exception exception) { + log.log(LogLevel.DEBUG, "Error reading port from script, using " + defaultPort); + return defaultPort; + } } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java index a2b7c7a5532..912809178ee 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -13,8 +13,8 @@ public class Slobrok extends AbstractService { private static final long serialVersionUID = 1L; /** - * @param parent The parent ConfigProducer. - * @param index unique index for all slobroks + * @param parent The parent ConfigProducer. + * @param index unique index for all slobroks */ public Slobrok(AbstractConfigProducer parent, int index) { super(parent, "slobrok." + index); @@ -42,8 +42,8 @@ public class Slobrok extends AbstractService { public String getStartupCommand() { return "exec $ROOT/bin/slobrok -p " + getPort() + - " -s " + getStatePort() + - " -c " + getConfigId(); + " -s " + getStatePort() + + " -c " + getConfigId(); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java index 8ec1cecf47a..d2dace21b86 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java @@ -17,4 +17,4 @@ public class ClusterControllerComponent extends SimpleComponent new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.ClusterController"), new ComponentSpecification("clustercontroller-apps")))); } -}
\ No newline at end of file +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java index e692b053de5..02e02cbe20a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java @@ -37,4 +37,4 @@ public class DeploymentFileValidator extends Validator { } } } -}
\ No newline at end of file +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 7017a5a233c..83d06f470a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -68,4 +68,4 @@ public class RankingConstantsValidator extends Validator { tensorApplicationFile.createReader()); } -}
\ No newline at end of file +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 56fa3d01abc..baf4b8d41e4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -48,18 +48,7 @@ public class Container extends AbstractService implements ComponentsConfig.Producer, JdiscBindingsConfig.Producer, ContainerHttpConfig.Producer, - ContainerMbusConfig.Producer -{ - public static final class PortOverride { - public final ComponentSpecification serverId; - public final int port; - - public PortOverride(ComponentSpecification serverId, int port) { - this.serverId = serverId; - this.port = port; - } - } - + ContainerMbusConfig.Producer { public static final int BASEPORT = Defaults.getDefaults().vespaWebServicePort(); public static final String SINGLENODE_CONTAINER_SERVICESPEC = "default_singlenode_container"; @@ -80,15 +69,15 @@ public class Container extends AbstractService implements private final int index; private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); - private final ComponentGroup<Component<?, ?>> components = new ComponentGroup(this, "components"); + private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); private final JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); private final List<PortOverride> portOverrides; private final int numHttpServerPorts; - private final int numRpcServerPorts = 2; - private static String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; + private static final int numRpcServerPorts = 2; + private static final String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; public Container(AbstractConfigProducer parent, String name, int index) { this(parent, name, Collections.<PortOverride>emptyList(), index); @@ -218,7 +207,7 @@ public class Container extends AbstractService implements return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null); } - // TODO: hack to retain old service names, e.g. in ymon config, vespa.log etc. + // TODO: hack to retain old service names, e.g. in monitoring config, vespa.log etc. @Override public String getServiceType() { if (parent instanceof ContainerCluster) { @@ -404,4 +393,14 @@ public class Container extends AbstractService implements this.httpServerEnabled = httpServerEnabled; } + public static final class PortOverride { + public final ComponentSpecification serverId; + public final int port; + + public PortOverride(ComponentSpecification serverId, int port) { + this.serverId = serverId; + this.port = port; + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index dbae6851977..fdb0f08f631 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -154,7 +154,7 @@ public final class ContainerCluster public static final String ROOT_HANDLER_BINDING = "*://*/"; - private String name; + private final String name; private List<Container> containers = new ArrayList<>(); @@ -243,9 +243,9 @@ public final class ContainerCluster Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT, - "https://*" + StateHandler.STATE_API_ROOT, - "http://*" + StateHandler.STATE_API_ROOT + "/*", - "https://*" + StateHandler.STATE_API_ROOT + "/*"); + "https://*" + StateHandler.STATE_API_ROOT, + "http://*" + StateHandler.STATE_API_ROOT + "/*", + "https://*" + StateHandler.STATE_API_ROOT + "/*"); addComponent(stateHandler); } @@ -308,12 +308,6 @@ public final class ContainerCluster } } - public final void addComponents(Collection<Component<?, ?>> components) { - for (Component<?, ?> component : components) { - addComponent(component); - } - } - public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); } @@ -847,14 +841,14 @@ public final class ContainerCluster } public static class MbusParams { - //the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) - public final Double maxConcurrentFactor; + // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) + final Double maxConcurrentFactor; - //the amount that documents expand temporarily when processing them - public final Double documentExpansionFactor; + // the amount that documents expand temporarily when processing them + final Double documentExpansionFactor; - //the space to reserve for container, docproc stuff (memory that cannot be used for processing documents), in MB - public final Integer containerCoreMemory; + // the space to reserve for container, docproc stuff (memory that cannot be used for processing documents), in MB + final Integer containerCoreMemory; public MbusParams(Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { this.maxConcurrentFactor = maxConcurrentFactor; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java index 304a6cd5052..e1671c78a94 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.vespa.model.container.http; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 90ad0117dfb..1ddfbbd76e1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -6,6 +6,7 @@ import com.google.common.collect.Sets; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; @@ -46,6 +47,7 @@ import org.w3c.dom.Element; import java.util.*; import java.util.logging.Level; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A content cluster. @@ -100,7 +102,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri ModelElement documentsElement = contentElement.getChild("documents"); Map<String, NewDocumentType> documentDefinitions = - new SearchDefinitionBuilder().build(context.getParentProducer().getRoot().getDeployState().getDocumentModel().getDocumentManager(), documentsElement); + new SearchDefinitionBuilder().build(context.getDeployState().getDocumentModel().getDocumentManager(), documentsElement); String routingSelection = new DocumentSelectionBuilder().build(documentsElement); Redundancy redundancy = new RedundancyBuilder().build(contentElement); @@ -108,7 +110,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterName(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, redundancy, - context.getParentProducer().getRoot().getDeployState().getProperties().zone()); + context.getDeployState().getProperties().zone()); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments).build(c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); @@ -585,19 +587,27 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri /** * Returns the distribution bits this cluster should use. - * OnHosted Vespa this is hardcoded not computed from the nodes because reducing the number of nodes is a common - * operation while reducing the number of distribution bits can lead to consistency problems. + * On Hosted Vespa this is hardcoded and not computed from the nodes because reducing the number of nodes is a common + * operation, while reducing the number of distribution bits can lead to consistency problems. * This hardcoded value should work fine from 1-200 nodes. Those who have more will need to set this value * in config and not remove it again if they reduce the node count. */ public int distributionBits() { - if (zone.region().equals(RegionName.from("us-west-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) - if (zone.region().equals(RegionName.from("us-central-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) - if (zone.region().equals(RegionName.from("eu-west-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) - if (zone.region().equals(RegionName.from("ap-northeast-1"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) - if (zone.region().equals(RegionName.from("ap-northeast-2"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) - if (zone.region().equals(RegionName.from("us-east-3"))) return 16; // TODO: Enable for all hosted zones (i.e when zone isn't default) - return DistributionBitCalculator.getDistributionBits(getNodeCountPerGroup(), getDistributionMode()); + // TODO: Enable for all hosted zones (i.e when zone isn't default) + List<Zone> zonesWith16DistributionBits = Arrays.asList(createZone(Environment.prod, "us-west-1"), + createZone(Environment.prod, "us-central-1"), + createZone(Environment.prod, "eu-west-1"), + createZone(Environment.prod, "ap-northeast-1"), + createZone(Environment.prod, "ap-northeast-2"), + createZone(Environment.prod, "us-east-3")); + if (zonesWith16DistributionBits.contains(zone)) + return 16; + else + return DistributionBitCalculator.getDistributionBits(getNodeCountPerGroup(), getDistributionMode()); + } + + private Zone createZone(Environment environment, String region) { + return new Zone(environment, RegionName.from(region)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 84685ecef3d..f43f579c26b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -147,4 +147,4 @@ public class FileSender implements Serializable { builder.setValue(reference.value()); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml b/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml index 66af18ed7bc..be6672f415f 100644 --- a/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml @@ -14,4 +14,4 @@ <node hostalias='node1' distribution-key='0'/> </nodes> </content> -</services>
\ No newline at end of file +</services> diff --git a/config-model/src/test/examples/weightedset-summaryto.sd b/config-model/src/test/examples/weightedset-summaryto.sd index b72d674e743..adc0ff163ea 100644 --- a/config-model/src/test/examples/weightedset-summaryto.sd +++ b/config-model/src/test/examples/weightedset-summaryto.sd @@ -12,4 +12,4 @@ search test { } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index b790d9e68e8..b7eb1414f6d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -35,4 +35,4 @@ public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); new DerivedConfiguration(search, rankProfileRegistry); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index 718f4592709..053d2ba87f2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -276,4 +276,4 @@ public class ConstantTensorJsonValidatorTest { "}")); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java index c2aee71063d..d126dd0585c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java @@ -25,4 +25,4 @@ public class RankingConstantsValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 0dea99c7b01..3ad3f2c2fdc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -163,4 +163,4 @@ public class IndexingScriptChangeValidatorTest { validate()); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 859dd15e040..9b0e02d580b 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -207,7 +207,7 @@ public class ContainerClusterTest { addContainer(cluster, "c2", "host-c2"); assertTrue(false); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().startsWith("Cluster container1 does not accept container com.yahoo.vespa.model.container.Container")); + assertEquals("Cluster container1 does not accept container qrserver on host 'host-c2'", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java index 126fcf7a583..82cbc338a8e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java @@ -1,8 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; @@ -14,15 +20,20 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.ProtonEngine; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.ContentClusterUtils; +import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; import java.util.List; +import static junit.framework.TestCase.assertEquals; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; +// TODO Rename to ContentClusterTest public class ClusterTest extends ContentBaseTest { private final static String HOSTS = "<admin version='2.0'><adminserver hostalias='mockhost' /></admin>"; @@ -412,20 +423,8 @@ public class ClusterTest extends ContentBaseTest { "</content>" ); - { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - cluster.getConfig(builder); - cluster.getClusterControllerConfig().getConfig(builder); - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); - assertEquals(8, config.ideal_distribution_bits()); - } + assertDistributionBitsInConfig(cluster, 8); - { - StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); - cluster.getConfig(builder); - StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); - assertEquals(8, config.minsplitcount()); - } cluster = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <documents/>" + @@ -438,20 +437,7 @@ public class ClusterTest extends ContentBaseTest { "</content>" ); - { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - cluster.getConfig(builder); - cluster.getClusterControllerConfig().getConfig(builder); - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); - assertEquals(8, config.ideal_distribution_bits()); - } - - { - StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); - cluster.getConfig(builder); - StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); - assertEquals(8, config.minsplitcount()); - } + assertDistributionBitsInConfig(cluster, 8); } @Test @@ -469,20 +455,8 @@ public class ClusterTest extends ContentBaseTest { "</content>" ); - { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - cluster.getConfig(builder); - cluster.getClusterControllerConfig().getConfig(builder); - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); - assertEquals(8, config.ideal_distribution_bits()); - } + assertDistributionBitsInConfig(cluster, 8); - { - StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); - cluster.getConfig(builder); - StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); - assertEquals(8, config.minsplitcount()); - } cluster = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <documents/>" + @@ -498,23 +472,20 @@ public class ClusterTest extends ContentBaseTest { "</content>" ); - { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); - cluster.getConfig(builder); - cluster.getClusterControllerConfig().getConfig(builder); - FleetcontrollerConfig config = new FleetcontrollerConfig(builder); - assertEquals(8, config.ideal_distribution_bits()); - } - - { - StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); - cluster.getConfig(builder); - StorDistributormanagerConfig config = new StorDistributormanagerConfig(builder); - assertEquals(8, config.minsplitcount()); - } + assertDistributionBitsInConfig(cluster, 8); } @Test + public void testZoneDependentDistributionBits() throws Exception { + String xml = new ContentClusterBuilder().docTypes("test").getXml(); + + ContentCluster prodWith16Bits = createWithZone(xml, new Zone(Environment.prod, RegionName.from("us-east-3"))); + assertDistributionBitsInConfig(prodWith16Bits, 16); + + ContentCluster stagingNot16Bits = createWithZone(xml, new Zone(Environment.staging, RegionName.from("us-east-3"))); + assertDistributionBitsInConfig(stagingNot16Bits, 8); + } + @Test public void testGenerateSearchNodes() { ContentCluster cluster = parse( @@ -818,4 +789,31 @@ public class ClusterTest extends ContentBaseTest { assertThat(cluster.getSearch().getSearchNodes().size(), is(1)); assertTrue(cluster.getSearch().getSearchNodes().get(0).getPreShutdownCommand().isPresent()); } + + private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new DeployProperties.Builder() + .hostedVespa(true) + .zone(zone) + .build()); + List<String> searchDefinitions = SearchDefinitionBuilder.createSearchDefinitions("test"); + MockRoot root = ContentClusterUtils.createMockRoot(searchDefinitions, deployStateBuilder); + ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root); + root.freezeModelTopology(); + cluster.validate(); + return cluster; + } + + private void assertDistributionBitsInConfig(ContentCluster cluster, int distributionBits) { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + FleetcontrollerConfig config = new FleetcontrollerConfig(builder); + assertEquals(distributionBits, config.ideal_distribution_bits()); + + StorDistributormanagerConfig.Builder sdBuilder = new StorDistributormanagerConfig.Builder(); + cluster.getConfig(sdBuilder); + StorDistributormanagerConfig storDistributormanagerConfig = new StorDistributormanagerConfig(sdBuilder); + assertEquals(distributionBits, storDistributormanagerConfig.minsplitcount()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 595013c506e..aebe1b76644 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.model.content.utils; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.HostProvisioner; -import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; @@ -34,14 +32,17 @@ public class ContentClusterUtils { } private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions) { - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build(); - DeployState deployState = new DeployState.Builder() - .applicationPackage(applicationPackage) - .modelHostProvisioner(provisioner) - .build(); - return new MockRoot("", deployState); + return createMockRoot(provisioner, searchDefinitions, new DeployState.Builder()); + } + private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build(); + deployStateBuilder.applicationPackage(applicationPackage) + .modelHostProvisioner(provisioner) + .build(); + return new MockRoot("", deployStateBuilder.build()); } + public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) throws Exception { return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions); } @@ -50,10 +51,14 @@ public class ContentClusterUtils { return createMockRoot(new SingleNodeProvisioner(), searchDefinitions); } + public static MockRoot createMockRoot(List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { + return createMockRoot(new SingleNodeProvisioner(), searchDefinitions, deployStateBuilder); + } + public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception { Document doc = XML.getDocument(clusterXml); Admin admin = new Admin(root, new Yamas("vespa", 60), new Metrics(), Collections.emptyMap(), false); - ConfigModelContext context = ConfigModelContext.create(null, DeployState.createTestState(), null, root, null); + ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), null, root, null); return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java index d58e005e2ae..77019689e0a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java @@ -59,4 +59,4 @@ public class GenericServicesModelTest { assertThat(gsModel.serviceClusters().get(0).services().size(), is(1)); } -}
\ No newline at end of file +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java index 40dc134190b..d9f8800ef77 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/PortsMetaTestCase.java @@ -34,4 +34,4 @@ public class PortsMetaTestCase extends junit.framework.TestCase { assertNull(pm.getHttpStatusOffset()); } -}
\ No newline at end of file +} diff --git a/config-provisioning/src/main/resources/configdefinitions/flavors.def b/config-provisioning/src/main/resources/configdefinitions/flavors.def index 3ba4c92bdc7..f86421b9695 100644 --- a/config-provisioning/src/main/resources/configdefinitions/flavors.def +++ b/config-provisioning/src/main/resources/configdefinitions/flavors.def @@ -47,4 +47,4 @@ flavor[].description string default="" flavor[].retired bool default=false # The free capacity we would like to preserve for this flavor -flavor[].idealHeadroom int default=0
\ No newline at end of file +flavor[].idealHeadroom int default=0 diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index b805640de87..b4005730ca4 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -24,4 +24,4 @@ interface ConfigSourceClient { String getActiveSourceConnection(); List<String> getSourceConnections(); -}
\ No newline at end of file +} diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java index 499772c60f4..04c2a84e1a3 100644 --- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java @@ -72,4 +72,4 @@ public class GenericConfigSubscriberTest { GenericConfigSubscriber sub = new GenericConfigSubscriber(); sub.subscribe(null, null, ""); } -}
\ No newline at end of file +} diff --git a/config/src/tests/configparser/configparser.cpp b/config/src/tests/configparser/configparser.cpp index 7a66d831ee5..521df96af3a 100644 --- a/config/src/tests/configparser/configparser.cpp +++ b/config/src/tests/configparser/configparser.cpp @@ -108,14 +108,6 @@ TEST("require that escaped values are properly unescaped") { ASSERT_EQUAL("a\nb\rc\\d\"eBg", value); } -IGNORE_TEST("verify that locale affects double parsing") { // Failing on some CentOS based environments - std::vector<vespalib::string> payload; - setlocale(LC_NUMERIC, "nb_NO.UTF-8"); - payload.push_back("foo 3.14"); - ASSERT_EXCEPTION(ConfigParser::parse<double>("foo", payload), InvalidConfigException, "Value 3.14 is not a legal double"); - setlocale(LC_NUMERIC, "C"); -} - TEST("require that maps can be parsed") { writeFile("foo.cfg", "\nfooValue \"a\"\nfooMap{\"foo\"} 1336\nfooMap{\"bar\"} 1337\n"); diff --git a/configd/AUTHORS b/configd/AUTHORS deleted file mode 100644 index f7c84236a0f..00000000000 --- a/configd/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -Finn Arne Gangstad <finnag@yahoo-inc.com> -Lars Christian Jensen <Lars.Jensen@europe.yahoo-inc.com> -Arne Henrik Juul <arnej27959> diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java index ed363c90a86..61a33dd430d 100644 --- a/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java +++ b/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java @@ -3,4 +3,4 @@ package com.yahoo.vespa.configdefinition; import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/pom.xml b/configserver/pom.xml index 0351000b760..519da2e80f2 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -239,7 +239,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java index ae72c61bcdb..268c6478d79 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java @@ -30,4 +30,4 @@ public class RefeedActionsFormatter { return builder.toString(); } -}
\ No newline at end of file +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index d5313db1495..a20d4e0c256 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -46,4 +46,4 @@ public class HttpGetConfigHandler extends HttpHandler { if (config == null) HttpConfigRequest.throwModelNotReady(); return config; } -}
\ No newline at end of file +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java index 1e7114957e9..73f2136e93a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java @@ -54,4 +54,4 @@ public class StatusResource { public StatusInformation getStatus() { return new StatusInformation(configserverConfig, modelVersions); } -}
\ No newline at end of file +} diff --git a/configserver/src/test/apps/validationOverride/services.xml b/configserver/src/test/apps/validationOverride/services.xml index c6779bf311e..9c652bff6ab 100644 --- a/configserver/src/test/apps/validationOverride/services.xml +++ b/configserver/src/test/apps/validationOverride/services.xml @@ -1,4 +1,4 @@ <jdisc version="1.0"> <search/> <nodes count="2"/> -</jdisc>
\ No newline at end of file +</jdisc> diff --git a/configserver/src/test/apps/validationOverride/validation-overrides.xml b/configserver/src/test/apps/validationOverride/validation-overrides.xml index 1f209d21c94..11c2cf72d5c 100644 --- a/configserver/src/test/apps/validationOverride/validation-overrides.xml +++ b/configserver/src/test/apps/validationOverride/validation-overrides.xml @@ -1,3 +1,3 @@ <validation-overrides> <allow until="2016-10-10">skip-old-config-models</allow> -</validation-overrides>
\ No newline at end of file +</validation-overrides> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java index cf4dda7d090..720008aa8f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java @@ -43,4 +43,4 @@ public class RefeedActionsFormatterTest { new RefeedActionsFormatter(actions).format()); } -}
\ No newline at end of file +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java index 9bcc462035a..2c2f7fc06b5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java @@ -27,4 +27,4 @@ public class ListTenantsResponseTest extends TenantTest { final ListTenantsResponse response = new ListTenantsResponse(tenantNames); assertResponseEquals(response, "{\"tenants\":[]}"); } -}
\ No newline at end of file +} diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/AccessLogSamplerTest.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/AccessLogSamplerTest.java index 33a4935d223..6e9191e88d6 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/AccessLogSamplerTest.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/AccessLogSamplerTest.java @@ -60,4 +60,4 @@ public class AccessLogSamplerTest { accessLogEntry.setURI(URI.create(uri)); return accessLogEntry; } -}
\ No newline at end of file +} diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java index d872f62ea82..c58d0e073e9 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java @@ -39,4 +39,4 @@ public class CircularArrayAccessLogKeeperTest { circularArrayAccessLogKeeper.addUri("b"); assertThat(circularArrayAccessLogKeeper.getUris(), contains("a", "b", "b")); } -}
\ No newline at end of file +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java index b92dc098004..a3959868c2a 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java @@ -9,4 +9,4 @@ import com.yahoo.osgi.annotation.ExportPackage; * and makes these available for in-process consumption through * {@link com.yahoo.container.jdisc.state.StateMonitor#snapshot}, * and off-process through a jDisc handler. - */
\ No newline at end of file + */ diff --git a/container-core/src/main/java/com/yahoo/metrics/package-info.java b/container-core/src/main/java/com/yahoo/metrics/package-info.java index 50374144683..29c45d043f7 100644 --- a/container-core/src/main/java/com/yahoo/metrics/package-info.java +++ b/container-core/src/main/java/com/yahoo/metrics/package-info.java @@ -3,4 +3,4 @@ @ExportPackage package com.yahoo.metrics; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java index ad266a4a87f..db8c44e72fd 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/AccessLogRequestHandlerTest.java @@ -44,4 +44,4 @@ public class AccessLogRequestHandlerTest { assertThat(out.toString(), is("{\"entries\":[{\"url\":\"foo\"},{\"url\":\"foo\"}]}")); } -}
\ No newline at end of file +} diff --git a/container-dev-builder/tools/src/main/java/com/yahoo/container/dev/builder/DependencyResolver.java b/container-dev-builder/tools/src/main/java/com/yahoo/container/dev/builder/DependencyResolver.java index c8a0bcf5a8a..7e9d7afaed3 100644 --- a/container-dev-builder/tools/src/main/java/com/yahoo/container/dev/builder/DependencyResolver.java +++ b/container-dev-builder/tools/src/main/java/com/yahoo/container/dev/builder/DependencyResolver.java @@ -55,4 +55,4 @@ public class DependencyResolver { System.out.println(dependency); } } -}
\ No newline at end of file +} diff --git a/container-di/src/main/scala/com/yahoo/container/di/package.scala b/container-di/src/main/scala/com/yahoo/container/di/package.scala index ec621af3830..11b02bfdf43 100644 --- a/container-di/src/main/scala/com/yahoo/container/di/package.scala +++ b/container-di/src/main/scala/com/yahoo/container/di/package.scala @@ -38,4 +38,4 @@ package object di { //For debug purposes only val preserveStackTrace: Boolean = Option(System.getProperty("jdisc.container.preserveStackTrace")).filterNot(_.isEmpty).isDefined -}
\ No newline at end of file +} diff --git a/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala b/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala index 20e4fcb03f8..fa6265ecf8e 100644 --- a/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala +++ b/container-di/src/test/scala/com/yahoo/container/di/DirConfigSource.scala @@ -46,4 +46,4 @@ private object DirConfigSource { folder.create() folder } -}
\ No newline at end of file +} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/package-info.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/package-info.java index 2731740abd5..6e046e570fe 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/package-info.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.container.jdisc.metric.state; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-disc/src/main/java/com/yahoo/jdisc/metrics/yamasconsumer/cloud/package-info.java b/container-disc/src/main/java/com/yahoo/jdisc/metrics/yamasconsumer/cloud/package-info.java index 9c3976cd972..ba7767414ef 100644 --- a/container-disc/src/main/java/com/yahoo/jdisc/metrics/yamasconsumer/cloud/package-info.java +++ b/container-disc/src/main/java/com/yahoo/jdisc/metrics/yamasconsumer/cloud/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.jdisc.metrics.yamasconsumer.cloud; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java b/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java index d29bc8780e8..b299dcf657e 100644 --- a/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java +++ b/container-search/src/main/java/com/yahoo/fs4/mplex/Backend.java @@ -403,4 +403,4 @@ public class Backend implements ConnectionFactory { return port; } -}
\ No newline at end of file +} diff --git a/container-search/src/main/java/com/yahoo/search/cache/package-info.java b/container-search/src/main/java/com/yahoo/search/cache/package-info.java index 292b491c52b..c28a75c2ccb 100644 --- a/container-search/src/main/java/com/yahoo/search/cache/package-info.java +++ b/container-search/src/main/java/com/yahoo/search/cache/package-info.java @@ -7,4 +7,4 @@ @ExportPackage package com.yahoo.search.cache; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg index 5f54d47353f..306e590a7ca 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg @@ -6,7 +6,7 @@ tokenlist[0].tokens[1].token c++ tokenlist[0].tokens[2].token b.s.d. tokenlist[0].tokens[3].token with space tokenlist[0].tokens[4].token c# -tokenlist[0].tokens[5].token dvd±r +tokenlist[0].tokens[5].token dvd\xB1r tokenlist[1].name other tokenlist[1].tokens[4] tokenlist[1].tokens[0].token [huh] diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java index a5ae6b78d4b..654347172af 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/PredicateQueryItemTestCase.java @@ -133,4 +133,4 @@ public class PredicateQueryItemTestCase { assertArrayEquals(expectedPrefix, Arrays.copyOfRange(actual, 0, expectedPrefix.length)); } -}
\ No newline at end of file +} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/templaterc b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/templaterc deleted file mode 100644 index 7654dd9fd2b..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/templaterc +++ /dev/null @@ -1,5 +0,0 @@ -# -# $Id: templaterc,v 1.1 2004-02-12 12:44:54 bratseth Exp $ -# -/xsearch xsearch text/xml utf-8 -/cgi-bin/asearch asearch text/html utf-8 diff --git a/dist/vespa.spec b/dist/vespa.spec index 3093f261006..1b4bc307468 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -31,10 +31,10 @@ BuildRequires: java-1.8.0-openjdk-devel BuildRequires: openssl-devel BuildRequires: rpm-build BuildRequires: make -BuildRequires: vespa-boost-devel >= 1.59 -BuildRequires: vespa-cppunit-devel >= 1.12.1 -BuildRequires: vespa-libtorrent-devel >= 1.0.9 -BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.8 +BuildRequires: vespa-boost-devel >= 1.59.0-6 +BuildRequires: vespa-cppunit-devel >= 1.12.1-6 +BuildRequires: vespa-libtorrent-devel >= 1.0.11-6 +BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.9-6 BuildRequires: systemd Requires: epel-release Requires: Judy @@ -45,10 +45,10 @@ Requires: libicu Requires: llvm3.9 Requires: java-1.8.0-openjdk Requires: openssl -Requires: vespa-boost >= 1.59 -Requires: vespa-cppunit >= 1.12.1 -Requires: vespa-libtorrent >= 1.0.9 -Requires: vespa-zookeeper-c-client >= 3.4.8 +Requires: vespa-boost >= 1.59.0-6 +Requires: vespa-cppunit >= 1.12.1-6 +Requires: vespa-libtorrent >= 1.0.11-6 +Requires: vespa-zookeeper-c-client >= 3.4.9-6 Requires(pre): shadow-utils # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java index f6b398e83a2..7ea6e597aed 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java @@ -36,4 +36,4 @@ public class CounterWrapper implements MetricValue { return value; } } -}
\ No newline at end of file +} diff --git a/docker-api/src/main/resources/configdefinitions/docker.def b/docker-api/src/main/resources/configdefinitions/docker.def index 5242e9f432c..9a5c9222afc 100644 --- a/docker-api/src/main/resources/configdefinitions/docker.def +++ b/docker-api/src/main/resources/configdefinitions/docker.def @@ -13,4 +13,4 @@ connectTimeoutMillis int default = 100000 # 100 sec readTimeoutMillis int default = 1800000 # 30 min isRunningLocally bool default = false -imageGCMinTimeToLiveMinutes int default = 45
\ No newline at end of file +imageGCMinTimeToLiveMinutes int default = 45 diff --git a/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py b/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py index 3edb0086f18..d148eefb899 100644 --- a/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py +++ b/docker-api/src/test/resources/simple-ipv6-server/src/fillmem.py @@ -8,4 +8,4 @@ data = megabyte * int(sys.argv[1]) while True: time.sleep(1) - data.extend(megabyte)
\ No newline at end of file + data.extend(megabyte) diff --git a/docker/.gitignore b/docker/.gitignore index e7a9c1347be..d8d0fbf70f1 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1 +1,2 @@ *.rpm +vespa-ci-*.log diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build index eb38354c84b..9aa35f2bd99 100644 --- a/docker/Dockerfile.build +++ b/docker/Dockerfile.build @@ -14,6 +14,7 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp ccache \ Judy-devel \ lz4-devel \ + libzstd-devel \ zlib-devel \ maven \ libicu-devel \ @@ -27,6 +28,10 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp perl-Env \ perl-JSON \ perl-IO-Socket-IP \ + perl-Data-Dumper \ + perl-libwww-perl \ + perl-Net-INET6Glue \ + perl-URI \ sudo \ vespa-boost-devel \ vespa-libtorrent-devel \ diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run index 5f0dd1fd0cc..d62faa5075c 100644 --- a/docker/Dockerfile.run +++ b/docker/Dockerfile.run @@ -3,24 +3,5 @@ FROM centos:7 # Needed to run vespa RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo && \ yum -y install epel-release && \ - yum -y install centos-release-scl && \ - yum -y --enablerepo=epel-testing install \ - Judy \ - lz4 \ - zlib \ - libicu \ - llvm3.9 \ - java-1.8.0-openjdk \ - openssl \ - perl \ - perl-Env \ - perl-JSON \ - perl-IO-Socket-IP \ - libatomic \ - net-tools \ - less \ - vespa-boost \ - vespa-libtorrent \ - vespa-zookeeper-c-client \ - vespa-cppunit + yum -y install centos-release-scl diff --git a/docker/run-vespa-internal.sh b/docker/run-vespa-internal.sh index dbfc9cc9777..b8c9af01611 100755 --- a/docker/run-vespa-internal.sh +++ b/docker/run-vespa-internal.sh @@ -12,7 +12,7 @@ cd $DIR VESPA_VERSION=$1 -rpm -i "vespa*-${VESPA_VERSION}-*.rpm" +yum localinstall -y $(ls vespa*-${VESPA_VERSION}-*.rpm | xargs) # Workaround until we figure out why rpm does not set the ownership. chown -R vespa:vespa /opt/vespa diff --git a/docker/vespa-ci-internal.sh b/docker/vespa-ci-internal.sh index 3d7cc439528..caa64824044 100755 --- a/docker/vespa-ci-internal.sh +++ b/docker/vespa-ci-internal.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -x if [ $# -ne 1 ]; then echo "Usage: $0 <git commit>" @@ -10,16 +11,17 @@ fi GIT_COMMIT=$1 SOURCE_DIR=/home/vespabuilder/vespa BUILD_DIR=/home/vespabuilder/build -THREADS=$(nproc --all) +NUM_CORES=$(nproc --all) +NUM_THREADS=$((${NUM_CORES} + ${NUM_CORES}/2)) mkdir "${SOURCE_DIR}" mkdir "${BUILD_DIR}" git clone --no-checkout --local --no-hardlinks file:///vespa "${SOURCE_DIR}" cd "${SOURCE_DIR}" -git checkout ${GIT_COMMIT} +git checkout --detach ${GIT_COMMIT} source /opt/rh/devtoolset-6/enable || true sh ./bootstrap.sh full -mvn install +MAVEN_OPTS="-Xms512m -Xmx512m" mvn install cd "${BUILD_DIR}" cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ @@ -28,5 +30,5 @@ cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \ -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \ "${SOURCE_DIR}" -make -j ${THREADS} -make -j ${THREADS} test +make -j ${NUM_THREADS} +make -j ${NUM_THREADS} test diff --git a/docker/vespa-ci.sh b/docker/vespa-ci.sh index 2d17634a409..58705eae339 100755 --- a/docker/vespa-ci.sh +++ b/docker/vespa-ci.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -x if [ $# -ne 1 ]; then echo "Usage: $0 <git commit>" @@ -15,4 +16,5 @@ CI_DOCKER_IMAGE="vespaci" docker build -t "$BUILD_DOCKER_IMAGE" -f Dockerfile.build . docker build -t "$CI_DOCKER_IMAGE" -f Dockerfile.ci . -docker run --rm -v $(pwd)/..:/vespa --entrypoint /vespa-ci-internal.sh "$CI_DOCKER_IMAGE" "$GIT_COMMIT" +docker run --rm -v $(pwd)/..:/vespa --entrypoint /vespa-ci-internal.sh "$CI_DOCKER_IMAGE" "$GIT_COMMIT" \ + 2>&1 | tee vespa-ci-${GIT_COMMIT}-$(date +%Y-%m-%dT%H:%M:%S%z).log diff --git a/docprocs/pom.xml b/docprocs/pom.xml index d85a4c475dd..b5c60b1179f 100644 --- a/docprocs/pom.xml +++ b/docprocs/pom.xml @@ -153,7 +153,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/document/AUTHORS b/document/AUTHORS deleted file mode 100644 index d88ca23dc13..00000000000 --- a/document/AUTHORS +++ /dev/null @@ -1,5 +0,0 @@ -thomasg -bratseth -fledsbo -borud -larschri diff --git a/document/src/main/java/com/yahoo/document/BucketDistribution.java b/document/src/main/java/com/yahoo/document/BucketDistribution.java index bb4792b5982..28ae01dd957 100644 --- a/document/src/main/java/com/yahoo/document/BucketDistribution.java +++ b/document/src/main/java/com/yahoo/document/BucketDistribution.java @@ -1,205 +1,205 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document;
-
-import com.yahoo.document.BucketId;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class BucketDistribution {
-
- // A logger object to enable proper logging.
- private static Logger log = Logger.getLogger(BucketDistribution.class.getName());
-
- // A map from bucket id to column index.
- private int[] bucketToColumn;
-
- // The number of columns to distribute to.
- private int numColumns;
-
- // The number of bits to use for bucket identification.
- private int numBucketBits;
-
- /**
- * Constructs a new bucket distribution object with a given number of columns and buckets.
- *
- * @param numColumns The number of columns to distribute to.
- * @param numBucketBits The number of bits to use for bucket id.
- */
- public BucketDistribution(int numColumns, int numBucketBits) {
- this.numBucketBits = numBucketBits;
- bucketToColumn = new int[getNumBuckets()];
- reset();
- setNumColumns(numColumns);
- }
-
- /**
- * Constructs a new bucket distribution object as a copy of another.
- *
- * @param other The distribution object to copy.
- */
- public BucketDistribution(BucketDistribution other) {
- bucketToColumn = other.bucketToColumn.clone();
- numColumns = other.numColumns;
- numBucketBits = other.numBucketBits;
- }
-
- /**
- * Returns the number of buckets that the given number of bucket bits will allow.
- *
- * @param numBucketBits The number of bits to use for bucket id.
- * @return The number of buckets allowed.
- */
- private static int getNumBuckets(int numBucketBits) {
- return 1 << numBucketBits;
- }
-
- /**
- * This method returns a list that contains the distributions of the given number of buckets over the given number
- * of columns.
- *
- * @param numColumns The number of columns to distribute to.
- * @param numBucketBits The number of bits to use for bucket id.
- * @return The bucket distribution.
- */
- private static List<Integer> getBucketCount(int numColumns, int numBucketBits) {
- List<Integer> ret = new ArrayList<Integer>(numColumns);
- int cnt = getNumBuckets(numBucketBits) / numColumns;
- int rst = getNumBuckets(numBucketBits) % numColumns;
- for (int i = 0; i < numColumns; ++i) {
- ret.add(cnt + (i < rst ? 1 : 0));
- }
- return ret;
- }
-
- /**
- * This method returns a list similar to {@link BucketDistribution#getBucketCount(int, int)}, except that the returned list
- * contains the number of buckets that will have to be migrated from each column if an additional column was added.
- *
- * @param numColumns The original number of columns.
- * @param numBucketBits The number of bits to use for bucket id.
- * @return The number of buckets to migrate, one value per column.
- */
- private static List<Integer> getBucketMigrateCount(int numColumns, int numBucketBits) {
- List<Integer> ret = getBucketCount(numColumns++, numBucketBits);
- int cnt = getNumBuckets(numBucketBits) / numColumns;
- int rst = getNumBuckets(numBucketBits) % numColumns;
- for (int i = 0; i < numColumns - 1; ++i) {
- ret.set(i, ret.get(i) - (cnt + (i < rst ? 1 : 0)));
- }
- return ret;
- }
-
- /**
- * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so
- * that it all buckets point to that single column.
- */
- public void reset() {
- for (int i = 0; i < bucketToColumn.length; ++i) {
- bucketToColumn[i] = 0;
- }
- numColumns = 1;
- }
-
- /**
- * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so
- * that it takes into account the new column.
- */
- private void addColumn() {
- int newColumns = numColumns + 1;
- List<Integer> migrate = getBucketMigrateCount(numColumns, numBucketBits);
- int numBuckets = getNumBuckets(numBucketBits);
- for (int i = 0; i < numBuckets; ++i) {
- int old = bucketToColumn[i];
- if (migrate.get(old) > 0) {
- bucketToColumn[i] = numColumns; // move this bucket to the new column
- migrate.set(old, migrate.get(old) - 1);
- }
- }
- numColumns = newColumns;
- }
-
- /**
- * Sets the number of columns to use for this document distribution object. This will reset and setup this object
- * from scratch. The original number of buckets is maintained.
- *
- * @param numColumns The new number of columns to distribute to.
- */
- public synchronized void setNumColumns(int numColumns) {
- if (numColumns < this.numColumns) {
- reset();
- }
- if (numColumns == this.numColumns) {
- return;
- }
- for (int i = numColumns - this.numColumns; --i >= 0; ) {
- addColumn();
- }
- }
-
- /**
- * Returns the number of columns to distribute to.
- *
- * @return The number of columns.
- */
- public int getNumColumns() {
- return numColumns;
- }
-
- /**
- * Sets the number of buckets to use for this document distribution object. This will reset and setup this object
- * from scratch. The original number of columns is maintained.
- *
- * @param numBucketBits The new number of bits to use for bucket id.
- */
- public synchronized void setNumBucketBits(int numBucketBits) {
- if (numBucketBits == this.numBucketBits) {
- return;
- }
- this.numBucketBits = numBucketBits;
- bucketToColumn = new int[getNumBuckets(numBucketBits)];
- int numColumns = this.numColumns;
- reset();
- setNumColumns(numColumns);
- }
-
- /**
- * Returns the number of bits used for bucket identifiers.
- *
- * @return The number of bits.
- */
- public int getNumBucketBits() {
- return numBucketBits;
- }
-
- /**
- * Returns the number of buckets available using the configured number of bucket bits.
- *
- * @return The number of buckets.
- */
- public int getNumBuckets() {
- return getNumBuckets(numBucketBits);
- }
-
- /**
- * This method maps the given bucket id to its corresponding column.
- *
- * @param bucketId The bucket whose column to lookup.
- * @return The column to distribute the bucket to.
- */
- public int getColumn(BucketId bucketId) {
- int ret = (int)(bucketId.getId() & (getNumBuckets(numBucketBits) - 1));
- if (ret >= bucketToColumn.length) {
- log.log(Level.SEVERE,
- "The bucket distribution map is not in sync with the number of bucket bits. " +
- "This should never happen! Distribution is broken!!");
- return 0;
- }
- return bucketToColumn[ret];
- }
-}
+package com.yahoo.document; + +import com.yahoo.document.BucketId; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class BucketDistribution { + + // A logger object to enable proper logging. + private static Logger log = Logger.getLogger(BucketDistribution.class.getName()); + + // A map from bucket id to column index. + private int[] bucketToColumn; + + // The number of columns to distribute to. + private int numColumns; + + // The number of bits to use for bucket identification. + private int numBucketBits; + + /** + * Constructs a new bucket distribution object with a given number of columns and buckets. + * + * @param numColumns The number of columns to distribute to. + * @param numBucketBits The number of bits to use for bucket id. + */ + public BucketDistribution(int numColumns, int numBucketBits) { + this.numBucketBits = numBucketBits; + bucketToColumn = new int[getNumBuckets()]; + reset(); + setNumColumns(numColumns); + } + + /** + * Constructs a new bucket distribution object as a copy of another. + * + * @param other The distribution object to copy. + */ + public BucketDistribution(BucketDistribution other) { + bucketToColumn = other.bucketToColumn.clone(); + numColumns = other.numColumns; + numBucketBits = other.numBucketBits; + } + + /** + * Returns the number of buckets that the given number of bucket bits will allow. + * + * @param numBucketBits The number of bits to use for bucket id. + * @return The number of buckets allowed. + */ + private static int getNumBuckets(int numBucketBits) { + return 1 << numBucketBits; + } + + /** + * This method returns a list that contains the distributions of the given number of buckets over the given number + * of columns. + * + * @param numColumns The number of columns to distribute to. + * @param numBucketBits The number of bits to use for bucket id. + * @return The bucket distribution. + */ + private static List<Integer> getBucketCount(int numColumns, int numBucketBits) { + List<Integer> ret = new ArrayList<Integer>(numColumns); + int cnt = getNumBuckets(numBucketBits) / numColumns; + int rst = getNumBuckets(numBucketBits) % numColumns; + for (int i = 0; i < numColumns; ++i) { + ret.add(cnt + (i < rst ? 1 : 0)); + } + return ret; + } + + /** + * This method returns a list similar to {@link BucketDistribution#getBucketCount(int, int)}, except that the returned list + * contains the number of buckets that will have to be migrated from each column if an additional column was added. + * + * @param numColumns The original number of columns. + * @param numBucketBits The number of bits to use for bucket id. + * @return The number of buckets to migrate, one value per column. + */ + private static List<Integer> getBucketMigrateCount(int numColumns, int numBucketBits) { + List<Integer> ret = getBucketCount(numColumns++, numBucketBits); + int cnt = getNumBuckets(numBucketBits) / numColumns; + int rst = getNumBuckets(numBucketBits) % numColumns; + for (int i = 0; i < numColumns - 1; ++i) { + ret.set(i, ret.get(i) - (cnt + (i < rst ? 1 : 0))); + } + return ret; + } + + /** + * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so + * that it all buckets point to that single column. + */ + public void reset() { + for (int i = 0; i < bucketToColumn.length; ++i) { + bucketToColumn[i] = 0; + } + numColumns = 1; + } + + /** + * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so + * that it takes into account the new column. + */ + private void addColumn() { + int newColumns = numColumns + 1; + List<Integer> migrate = getBucketMigrateCount(numColumns, numBucketBits); + int numBuckets = getNumBuckets(numBucketBits); + for (int i = 0; i < numBuckets; ++i) { + int old = bucketToColumn[i]; + if (migrate.get(old) > 0) { + bucketToColumn[i] = numColumns; // move this bucket to the new column + migrate.set(old, migrate.get(old) - 1); + } + } + numColumns = newColumns; + } + + /** + * Sets the number of columns to use for this document distribution object. This will reset and setup this object + * from scratch. The original number of buckets is maintained. + * + * @param numColumns The new number of columns to distribute to. + */ + public synchronized void setNumColumns(int numColumns) { + if (numColumns < this.numColumns) { + reset(); + } + if (numColumns == this.numColumns) { + return; + } + for (int i = numColumns - this.numColumns; --i >= 0; ) { + addColumn(); + } + } + + /** + * Returns the number of columns to distribute to. + * + * @return The number of columns. + */ + public int getNumColumns() { + return numColumns; + } + + /** + * Sets the number of buckets to use for this document distribution object. This will reset and setup this object + * from scratch. The original number of columns is maintained. + * + * @param numBucketBits The new number of bits to use for bucket id. + */ + public synchronized void setNumBucketBits(int numBucketBits) { + if (numBucketBits == this.numBucketBits) { + return; + } + this.numBucketBits = numBucketBits; + bucketToColumn = new int[getNumBuckets(numBucketBits)]; + int numColumns = this.numColumns; + reset(); + setNumColumns(numColumns); + } + + /** + * Returns the number of bits used for bucket identifiers. + * + * @return The number of bits. + */ + public int getNumBucketBits() { + return numBucketBits; + } + + /** + * Returns the number of buckets available using the configured number of bucket bits. + * + * @return The number of buckets. + */ + public int getNumBuckets() { + return getNumBuckets(numBucketBits); + } + + /** + * This method maps the given bucket id to its corresponding column. + * + * @param bucketId The bucket whose column to lookup. + * @return The column to distribute the bucket to. + */ + public int getColumn(BucketId bucketId) { + int ret = (int)(bucketId.getId() & (getNumBuckets(numBucketBits) - 1)); + if (ret >= bucketToColumn.length) { + log.log(Level.SEVERE, + "The bucket distribution map is not in sync with the number of bucket bits. " + + "This should never happen! Distribution is broken!!"); + return 0; + } + return bucketToColumn[ret]; + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java index 96a3fadc66c..3e911afb305 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java @@ -52,4 +52,4 @@ public class RemoveFieldPathUpdate extends FieldPathUpdate { public String toString() { return "Remove: " + super.toString(); } -}
\ No newline at end of file +} diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 1c5e6efdda6..36dbacfcde1 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -407,4 +407,4 @@ public class DocumentUpdateJsonSerializer return this; } } -}
\ No newline at end of file +} diff --git a/document/src/main/java/com/yahoo/document/select/BucketSet.java b/document/src/main/java/com/yahoo/document/select/BucketSet.java index e7bb4ac7807..286c66969d9 100644 --- a/document/src/main/java/com/yahoo/document/select/BucketSet.java +++ b/document/src/main/java/com/yahoo/document/select/BucketSet.java @@ -1,72 +1,72 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select;
-
-import com.yahoo.document.BucketId;
-
-import java.util.HashSet;
-
-/**
- * A set of bucket ids covered by a document selector.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class BucketSet extends HashSet<BucketId> {
-
- /**
- * Constructs a new bucket set that contains no ids.
- */
- public BucketSet() {
- // empty
- }
-
- /**
- * Constructs a new bucket set that contains a single id.
- *
- * @param id The id to add to this as initial value.
- */
- public BucketSet(BucketId id) {
- add(id);
- }
-
- /**
- * Constructs a new bucket set that is a copy of another.
- *
- * @param set The set to copy.
- */
- public BucketSet(BucketSet set) {
- this.addAll(set);
- }
-
- /**
- * Returns the intersection between this bucket set and another.
- *
- * @param rhs The set to form an intersection with.
- * @return The intersection.
- */
- public BucketSet intersection(BucketSet rhs) {
- if (rhs == null) {
- return new BucketSet(this); // The other has all buckets marked, this is the smaller.
- } else {
- BucketSet ret = new BucketSet(this);
- ret.retainAll(rhs);
- return ret;
- }
- }
-
- /**
- * Returns the union between this bucket set and another.
- *
- * @param rhs The set to form a union with.
- * @return The union.
- */
- public BucketSet union(BucketSet rhs) {
- if (rhs == null) {
- return null;
- } else {
- BucketSet ret = new BucketSet(this);
- ret.addAll(rhs);
- return ret;
- }
- }
-
-}
+package com.yahoo.document.select; + +import com.yahoo.document.BucketId; + +import java.util.HashSet; + +/** + * A set of bucket ids covered by a document selector. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class BucketSet extends HashSet<BucketId> { + + /** + * Constructs a new bucket set that contains no ids. + */ + public BucketSet() { + // empty + } + + /** + * Constructs a new bucket set that contains a single id. + * + * @param id The id to add to this as initial value. + */ + public BucketSet(BucketId id) { + add(id); + } + + /** + * Constructs a new bucket set that is a copy of another. + * + * @param set The set to copy. + */ + public BucketSet(BucketSet set) { + this.addAll(set); + } + + /** + * Returns the intersection between this bucket set and another. + * + * @param rhs The set to form an intersection with. + * @return The intersection. + */ + public BucketSet intersection(BucketSet rhs) { + if (rhs == null) { + return new BucketSet(this); // The other has all buckets marked, this is the smaller. + } else { + BucketSet ret = new BucketSet(this); + ret.retainAll(rhs); + return ret; + } + } + + /** + * Returns the union between this bucket set and another. + * + * @param rhs The set to form a union with. + * @return The union. + */ + public BucketSet union(BucketSet rhs) { + if (rhs == null) { + return null; + } else { + BucketSet ret = new BucketSet(this); + ret.addAll(rhs); + return ret; + } + } + +} diff --git a/document/src/main/java/com/yahoo/document/select/Result.java b/document/src/main/java/com/yahoo/document/select/Result.java index 3f1fa75d4ef..e0b8f02c47b 100644 --- a/document/src/main/java/com/yahoo/document/select/Result.java +++ b/document/src/main/java/com/yahoo/document/select/Result.java @@ -1,53 +1,53 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select;
-
-import com.yahoo.document.select.rule.AttributeNode;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public enum Result {
-
- /**
- * Defines all enumeration constants.
- */
- TRUE,
- FALSE,
- INVALID;
-
- // Inherit doc from Object.
- public String toString() {
- return name().toLowerCase();
- }
-
- /**
- * Inverts the result value to the appropriate value. True → False, False → True and Invalid → Invalid.
- * @return inverted result
- */
- public static Result invert(Result result) {
- if (result == Result.TRUE) return Result.FALSE;
- if (result == Result.FALSE) return Result.TRUE;
- return Result.INVALID;
- }
-
- /**
- * Converts the given object value into an instance of this Result enumeration.
- *
- * @param value The object to convert.
- * @return The corresponding result value.
- */
- public static Result toResult(Object value) {
- if (value == null || value == Result.FALSE || value == Boolean.FALSE ||
- (Number.class.isInstance(value) && ((Number)value).doubleValue() == 0)) {
- return Result.FALSE;
- } else if (value == INVALID) {
- return Result.INVALID;
- } else if (value instanceof AttributeNode.VariableValueList) {
- return ((AttributeNode.VariableValueList)value).isEmpty() ? Result.FALSE : Result.TRUE;
- } else if (value instanceof ResultList) {
- return ((ResultList)value).toResult();
- } else {
- return Result.TRUE;
- }
- }
-}
+package com.yahoo.document.select; + +import com.yahoo.document.select.rule.AttributeNode; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public enum Result { + + /** + * Defines all enumeration constants. + */ + TRUE, + FALSE, + INVALID; + + // Inherit doc from Object. + public String toString() { + return name().toLowerCase(); + } + + /** + * Inverts the result value to the appropriate value. True → False, False → True and Invalid → Invalid. + * @return inverted result + */ + public static Result invert(Result result) { + if (result == Result.TRUE) return Result.FALSE; + if (result == Result.FALSE) return Result.TRUE; + return Result.INVALID; + } + + /** + * Converts the given object value into an instance of this Result enumeration. + * + * @param value The object to convert. + * @return The corresponding result value. + */ + public static Result toResult(Object value) { + if (value == null || value == Result.FALSE || value == Boolean.FALSE || + (Number.class.isInstance(value) && ((Number)value).doubleValue() == 0)) { + return Result.FALSE; + } else if (value == INVALID) { + return Result.INVALID; + } else if (value instanceof AttributeNode.VariableValueList) { + return ((AttributeNode.VariableValueList)value).isEmpty() ? Result.FALSE : Result.TRUE; + } else if (value instanceof ResultList) { + return ((ResultList)value).toResult(); + } else { + return Result.TRUE; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java index 2fe4609b4e6..3bb558c2eb2 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java @@ -1,209 +1,209 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.datatypes.NumericFieldValue;
-import com.yahoo.document.select.*;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Stack;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class ArithmeticNode implements ExpressionNode {
-
- public static final int NOP = 0;
- public static final int ADD = 1;
- public static final int SUB = 2;
- public static final int MOD = 3;
- public static final int DIV = 4;
- public static final int MUL = 5;
-
- private final List<NodeItem> items = new ArrayList<NodeItem>();
-
- public ArithmeticNode() {
- // empty
- }
-
- public ArithmeticNode add(String operator, ExpressionNode node) {
- items.add(new NodeItem(stringToOperator(operator), node));
- return this;
- }
-
- public List<NodeItem> getItems() {
- return items;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public Object evaluate(Context context) {
- StringBuilder ret = null;
- Stack<ValueItem> buf = new Stack<ValueItem>();
- for (int i = 0; i < items.size(); ++i) {
- NodeItem item = items.get(i);
- Object val = item.node.evaluate(context);
-
- if (val == null) {
- throw new IllegalStateException("Null value found!");
- }
-
- if (val instanceof AttributeNode.VariableValueList) {
- AttributeNode.VariableValueList value = (AttributeNode.VariableValueList)val;
- if (value.size() == 0) {
- throw new IllegalArgumentException("Can not perform arithmetic on missing field: "
- + item.node.toString());
- } else if (value.size() != 1) {
- throw new IllegalStateException("Arithmetic is only valid for single values.");
- } else {
- val = value.get(0).getValue();
- }
- }
-
- if (val instanceof NumericFieldValue) {
- val = ((NumericFieldValue)val).getNumber();
- }
-
- if (val instanceof String) {
- if (i == 0) {
- ret = new StringBuilder();
- }
- if (ret != null) {
- ret.append(val);
- continue;
- }
- } else if (Number.class.isInstance(val)) {
- if (!buf.isEmpty()) {
- while (buf.peek().operator > item.operator) {
- popOffTheTop(buf);
- }
- }
- buf.push(new ValueItem(item.operator, (Number)val));
- continue;
- }
- throw new IllegalStateException("Term '" + item.node + " with class " + val.getClass() + "' does not evaluate to a number.");
- }
- if (ret != null) {
- return ret.toString();
- }
- while (buf.size() > 1) {
- popOffTheTop(buf);
- }
- return buf.pop().value;
- }
-
- private void popOffTheTop(Stack<ValueItem> buf) {
- ValueItem rhs = buf.pop();
- ValueItem lhs = buf.pop();
- switch (rhs.operator) {
- case ADD:
- lhs.value = lhs.value.doubleValue() + rhs.value.doubleValue();
- break;
- case SUB:
- lhs.value = lhs.value.doubleValue() - rhs.value.doubleValue();
- break;
- case DIV:
- lhs.value = lhs.value.doubleValue() / rhs.value.doubleValue();
- break;
- case MUL:
- lhs.value = lhs.value.doubleValue() * rhs.value.doubleValue();
- break;
- case MOD:
- lhs.value = lhs.value.longValue() % rhs.value.longValue();
- break;
- default:
- throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
- }
- buf.push(lhs);
- }
-
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- for (NodeItem item : items) {
- if (item.operator != NOP) {
- ret.append(" ").append(operatorToString(item.operator)).append(" ");
- }
- ret.append(item.node);
- }
- return ret.toString();
- }
-
- public String operatorToString(int operator) {
- switch (operator) {
- case NOP:
- return null;
- case ADD:
- return "+";
- case SUB:
- return "-";
- case MOD:
- return "%";
- case DIV:
- return "/";
- case MUL:
- return "*";
- default:
- throw new IllegalStateException("Arithmetic operator " + operator + " not supported.");
- }
- }
-
- private int stringToOperator(String operator) {
- if (operator == null) {
- return NOP;
- } else if (operator.equals("+")) {
- return ADD;
- } else if (operator.equals("-")) {
- return SUB;
- } else if (operator.equals("%")) {
- return MOD;
- } else if (operator.equals("/")) {
- return DIV;
- } else if (operator.equals("*")) {
- return MUL;
- } else {
- throw new IllegalStateException("Arithmetic operator '" + operator + "' not supported.");
- }
- }
-
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- private class ValueItem {
- public int operator;
- public Number value;
-
- public ValueItem(int operator, Number value) {
- this.operator = operator;
- this.value = value;
- }
- }
-
- public static class NodeItem {
- private int operator;
- private ExpressionNode node;
-
- public NodeItem(int operator, ExpressionNode node) {
- this.operator = operator;
- this.node = node;
- }
-
- public int getOperator() {
- return operator;
- }
-
- public ExpressionNode getNode() {
- return node;
- }
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.document.select.*; + +import java.util.List; +import java.util.ArrayList; +import java.util.Stack; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ArithmeticNode implements ExpressionNode { + + public static final int NOP = 0; + public static final int ADD = 1; + public static final int SUB = 2; + public static final int MOD = 3; + public static final int DIV = 4; + public static final int MUL = 5; + + private final List<NodeItem> items = new ArrayList<NodeItem>(); + + public ArithmeticNode() { + // empty + } + + public ArithmeticNode add(String operator, ExpressionNode node) { + items.add(new NodeItem(stringToOperator(operator), node)); + return this; + } + + public List<NodeItem> getItems() { + return items; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + // Inherit doc from ExpressionNode. + public Object evaluate(Context context) { + StringBuilder ret = null; + Stack<ValueItem> buf = new Stack<ValueItem>(); + for (int i = 0; i < items.size(); ++i) { + NodeItem item = items.get(i); + Object val = item.node.evaluate(context); + + if (val == null) { + throw new IllegalStateException("Null value found!"); + } + + if (val instanceof AttributeNode.VariableValueList) { + AttributeNode.VariableValueList value = (AttributeNode.VariableValueList)val; + if (value.size() == 0) { + throw new IllegalArgumentException("Can not perform arithmetic on missing field: " + + item.node.toString()); + } else if (value.size() != 1) { + throw new IllegalStateException("Arithmetic is only valid for single values."); + } else { + val = value.get(0).getValue(); + } + } + + if (val instanceof NumericFieldValue) { + val = ((NumericFieldValue)val).getNumber(); + } + + if (val instanceof String) { + if (i == 0) { + ret = new StringBuilder(); + } + if (ret != null) { + ret.append(val); + continue; + } + } else if (Number.class.isInstance(val)) { + if (!buf.isEmpty()) { + while (buf.peek().operator > item.operator) { + popOffTheTop(buf); + } + } + buf.push(new ValueItem(item.operator, (Number)val)); + continue; + } + throw new IllegalStateException("Term '" + item.node + " with class " + val.getClass() + "' does not evaluate to a number."); + } + if (ret != null) { + return ret.toString(); + } + while (buf.size() > 1) { + popOffTheTop(buf); + } + return buf.pop().value; + } + + private void popOffTheTop(Stack<ValueItem> buf) { + ValueItem rhs = buf.pop(); + ValueItem lhs = buf.pop(); + switch (rhs.operator) { + case ADD: + lhs.value = lhs.value.doubleValue() + rhs.value.doubleValue(); + break; + case SUB: + lhs.value = lhs.value.doubleValue() - rhs.value.doubleValue(); + break; + case DIV: + lhs.value = lhs.value.doubleValue() / rhs.value.doubleValue(); + break; + case MUL: + lhs.value = lhs.value.doubleValue() * rhs.value.doubleValue(); + break; + case MOD: + lhs.value = lhs.value.longValue() % rhs.value.longValue(); + break; + default: + throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); + } + buf.push(lhs); + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + for (NodeItem item : items) { + if (item.operator != NOP) { + ret.append(" ").append(operatorToString(item.operator)).append(" "); + } + ret.append(item.node); + } + return ret.toString(); + } + + public String operatorToString(int operator) { + switch (operator) { + case NOP: + return null; + case ADD: + return "+"; + case SUB: + return "-"; + case MOD: + return "%"; + case DIV: + return "/"; + case MUL: + return "*"; + default: + throw new IllegalStateException("Arithmetic operator " + operator + " not supported."); + } + } + + private int stringToOperator(String operator) { + if (operator == null) { + return NOP; + } else if (operator.equals("+")) { + return ADD; + } else if (operator.equals("-")) { + return SUB; + } else if (operator.equals("%")) { + return MOD; + } else if (operator.equals("/")) { + return DIV; + } else if (operator.equals("*")) { + return MUL; + } else { + throw new IllegalStateException("Arithmetic operator '" + operator + "' not supported."); + } + } + + public OrderingSpecification getOrdering(int order) { + return null; + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + private class ValueItem { + public int operator; + public Number value; + + public ValueItem(int operator, Number value) { + this.operator = operator; + this.value = value; + } + } + + public static class NodeItem { + private int operator; + private ExpressionNode node; + + public NodeItem(int operator, ExpressionNode node) { + this.operator = operator; + this.node = node; + } + + public int getOperator() { + return operator; + } + + public ExpressionNode getNode() { + return node; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java index 048eb70ac94..7e51834d871 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -1,205 +1,205 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.collections.BobHash;
-import com.yahoo.document.*;
-import com.yahoo.document.datatypes.FieldPathIteratorHandler;
-import com.yahoo.document.datatypes.FieldValue;
-import com.yahoo.document.select.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class AttributeNode implements ExpressionNode {
-
- private ExpressionNode value;
- private final List<Item> items = new ArrayList<Item>();
-
- public AttributeNode(ExpressionNode value, List items) {
- this.value = value;
- for (Object obj : items) {
- if (obj instanceof Item) {
- this.items.add((Item)obj);
- } else {
- throw new IllegalStateException("Can not add an instance of " + obj.getClass().getName() +
- " as a function item.");
- }
- }
- }
-
- public ExpressionNode getValue() {
- return value;
- }
-
- public AttributeNode setValue(ExpressionNode value) {
- this.value = value;
- return this;
- }
-
- public List<Item> getItems() {
- return items;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public Object evaluate(Context context) {
- String pos = value.toString();
- Object obj = value.evaluate(context);
-
- StringBuilder builder = new StringBuilder();
- for (Item item : items) {
- if (obj == null) {
- throw new IllegalStateException("Can not invoke '" + item + "' on '" + pos + "' because that term " +
- "evaluated to null.");
- }
- if (item.getType() != Item.FUNCTION) {
- if (builder.length() > 0) {
- builder.append(".");
- }
-
- builder.append(item.getName());
- } else {
- if (builder.length() > 0) {
- obj = evaluateFieldPath(builder.toString(), obj);
- builder = new StringBuilder();
- }
-
- obj = evaluateFunction(item.getName(), obj);
- }
-
- pos = pos + "." + item;
- }
-
- if (builder.length() > 0) {
- obj = evaluateFieldPath(builder.toString(), obj);
- }
- return obj;
- }
-
- public static class VariableValueList extends ArrayList<ResultList.VariableValue> {
-
- }
-
- static class IteratorHandler extends FieldPathIteratorHandler {
- VariableValueList values = new VariableValueList();
-
- @Override
- public void onPrimitive(FieldValue fv) {
- values.add(new ResultList.VariableValue((VariableMap)getVariables().clone(), fv));
- }
- }
-
- private static Object applyFunction(String function, Object value) {
- if (function.equalsIgnoreCase("abs")) {
- if (Number.class.isInstance(value)) {
- Number nValue = (Number)value;
- if (value instanceof Double) {
- return nValue.doubleValue() * (nValue.doubleValue() < 0 ? -1 : 1);
- } else if (value instanceof Float) {
- return nValue.floatValue() * (nValue.floatValue() < 0 ? -1 : 1);
- } else if (value instanceof Long) {
- return nValue.longValue() * (nValue.longValue() < 0 ? -1 : 1);
- } else if (value instanceof Integer) {
- return nValue.intValue() * (nValue.intValue() < 0 ? -1 : 1);
- }
- }
- throw new IllegalStateException("Function 'abs' is only available for numerical values.");
- } else if (function.equalsIgnoreCase("hash")) {
- return BobHash.hash(value.toString());
- } else if (function.equalsIgnoreCase("lowercase")) {
- return value.toString().toLowerCase();
- } else if (function.equalsIgnoreCase("uppercase")) {
- return value.toString().toUpperCase();
- }
- throw new IllegalStateException("Function '" + function + "' is not supported.");
- }
-
- private static Object evaluateFieldPath(String fieldPth, Object value) {
- if (value instanceof DocumentPut) {
- final Document doc = ((DocumentPut) value).getDocument();
- FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth);
- IteratorHandler handler = new IteratorHandler();
- doc.iterateNested(fieldPath, 0, handler);
- return handler.values;
- } else if (value instanceof DocumentUpdate) {
- return Result.INVALID;
- }
- return Result.FALSE;
- //throw new IllegalStateException("Attributes are only available for document types for value '" + value + "'. Looking for " + fieldPth);
- }
-
- private static Object evaluateFunction(String function, Object value) {
- if (value instanceof VariableValueList) {
- VariableValueList retVal = new VariableValueList();
-
- for (ResultList.VariableValue val : ((VariableValueList)value)) {
- retVal.add(new ResultList.VariableValue(
- (FieldPathIteratorHandler.VariableMap)val.getVariables().clone(),
- applyFunction(function, val.getValue())));
- }
-
- return retVal;
- }
-
- return applyFunction(function, value);
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- ret.append(value);
- for (Item item : items) {
- ret.append(".").append(item);
- }
- return ret.toString();
- }
-
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-
- public static class Item {
- public static final int ATTRIBUTE = 0;
- public static final int FUNCTION = 1;
-
- private String name;
- private int type = ATTRIBUTE;
-
- public Item(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
- public Item setName(String name) {
- this.name = name;
- return this;
- }
-
- public int getType() {
- return type;
- }
-
- public Item setType(int type) {
- this.type = type;
- return this;
- }
-
- @Override public String toString() {
- return name + (type == FUNCTION ? "()" : "");
- }
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.collections.BobHash; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.select.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class AttributeNode implements ExpressionNode { + + private ExpressionNode value; + private final List<Item> items = new ArrayList<Item>(); + + public AttributeNode(ExpressionNode value, List items) { + this.value = value; + for (Object obj : items) { + if (obj instanceof Item) { + this.items.add((Item)obj); + } else { + throw new IllegalStateException("Can not add an instance of " + obj.getClass().getName() + + " as a function item."); + } + } + } + + public ExpressionNode getValue() { + return value; + } + + public AttributeNode setValue(ExpressionNode value) { + this.value = value; + return this; + } + + public List<Item> getItems() { + return items; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + // Inherit doc from ExpressionNode. + public Object evaluate(Context context) { + String pos = value.toString(); + Object obj = value.evaluate(context); + + StringBuilder builder = new StringBuilder(); + for (Item item : items) { + if (obj == null) { + throw new IllegalStateException("Can not invoke '" + item + "' on '" + pos + "' because that term " + + "evaluated to null."); + } + if (item.getType() != Item.FUNCTION) { + if (builder.length() > 0) { + builder.append("."); + } + + builder.append(item.getName()); + } else { + if (builder.length() > 0) { + obj = evaluateFieldPath(builder.toString(), obj); + builder = new StringBuilder(); + } + + obj = evaluateFunction(item.getName(), obj); + } + + pos = pos + "." + item; + } + + if (builder.length() > 0) { + obj = evaluateFieldPath(builder.toString(), obj); + } + return obj; + } + + public static class VariableValueList extends ArrayList<ResultList.VariableValue> { + + } + + static class IteratorHandler extends FieldPathIteratorHandler { + VariableValueList values = new VariableValueList(); + + @Override + public void onPrimitive(FieldValue fv) { + values.add(new ResultList.VariableValue((VariableMap)getVariables().clone(), fv)); + } + } + + private static Object applyFunction(String function, Object value) { + if (function.equalsIgnoreCase("abs")) { + if (Number.class.isInstance(value)) { + Number nValue = (Number)value; + if (value instanceof Double) { + return nValue.doubleValue() * (nValue.doubleValue() < 0 ? -1 : 1); + } else if (value instanceof Float) { + return nValue.floatValue() * (nValue.floatValue() < 0 ? -1 : 1); + } else if (value instanceof Long) { + return nValue.longValue() * (nValue.longValue() < 0 ? -1 : 1); + } else if (value instanceof Integer) { + return nValue.intValue() * (nValue.intValue() < 0 ? -1 : 1); + } + } + throw new IllegalStateException("Function 'abs' is only available for numerical values."); + } else if (function.equalsIgnoreCase("hash")) { + return BobHash.hash(value.toString()); + } else if (function.equalsIgnoreCase("lowercase")) { + return value.toString().toLowerCase(); + } else if (function.equalsIgnoreCase("uppercase")) { + return value.toString().toUpperCase(); + } + throw new IllegalStateException("Function '" + function + "' is not supported."); + } + + private static Object evaluateFieldPath(String fieldPth, Object value) { + if (value instanceof DocumentPut) { + final Document doc = ((DocumentPut) value).getDocument(); + FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth); + IteratorHandler handler = new IteratorHandler(); + doc.iterateNested(fieldPath, 0, handler); + return handler.values; + } else if (value instanceof DocumentUpdate) { + return Result.INVALID; + } + return Result.FALSE; + //throw new IllegalStateException("Attributes are only available for document types for value '" + value + "'. Looking for " + fieldPth); + } + + private static Object evaluateFunction(String function, Object value) { + if (value instanceof VariableValueList) { + VariableValueList retVal = new VariableValueList(); + + for (ResultList.VariableValue val : ((VariableValueList)value)) { + retVal.add(new ResultList.VariableValue( + (FieldPathIteratorHandler.VariableMap)val.getVariables().clone(), + applyFunction(function, val.getValue()))); + } + + return retVal; + } + + return applyFunction(function, value); + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + ret.append(value); + for (Item item : items) { + ret.append(".").append(item); + } + return ret.toString(); + } + + public OrderingSpecification getOrdering(int order) { + return null; + } + + public static class Item { + public static final int ATTRIBUTE = 0; + public static final int FUNCTION = 1; + + private String name; + private int type = ATTRIBUTE; + + public Item(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public Item setName(String name) { + this.name = name; + return this; + } + + public int getType() { + return type; + } + + public Item setType(int type) { + this.type = type; + return this; + } + + @Override public String toString() { + return name + (type == FUNCTION ? "()" : ""); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index b0d5030978e..9db5ff0384b 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -1,435 +1,435 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketId;
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.datatypes.FieldPathIteratorHandler;
-import com.yahoo.document.datatypes.NumericFieldValue;
-import com.yahoo.document.idstring.GroupDocIdString;
-import com.yahoo.document.select.*;
-
-import java.util.List;
-import java.util.regex.Pattern;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class ComparisonNode implements ExpressionNode {
-
- // The left- and right-hand-side of this comparison.
- private ExpressionNode lhs, rhs;
-
- // The operator string for this.
- private String operator;
-
- /**
- * Constructs a new comparison node.
- *
- * @param lhs The left-hand-side of the comparison.
- * @param operator The comparison operator.
- * @param rhs The right-hand-side of the comparison.
- */
- public ComparisonNode(ExpressionNode lhs, String operator, ExpressionNode rhs) {
- this.lhs = lhs;
- this.operator = operator;
- this.rhs = rhs;
- }
-
- /**
- * Returns the left hand side of this comparison.
- *
- * @return The left hand side expression.
- */
- public ExpressionNode getLHS() {
- return lhs;
- }
-
- /**
- * Sets the left hand side of this comparison.
- *
- * @param lhs The new left hand side.
- * @return This, to allow chaining.
- */
- public ComparisonNode setLHS(ExpressionNode lhs) {
- this.lhs = lhs;
- return this;
- }
-
- /**
- * Returns the comparison operator of this.
- *
- * @return The operator.
- */
- public String getOperator() {
- return operator;
- }
-
- /**
- * Sets the comparison operator of this.
- *
- * @param operator The operator string.
- * @return This, to allow chaining.
- */
- public ComparisonNode setOperator(String operator) {
- this.operator = operator;
- return this;
- }
-
- /**
- * Returns the right hand side of this comparison.
- *
- * @return The right hand side expression.
- */
- public ExpressionNode getRHS() {
- return rhs;
- }
-
- /**
- * Sets the right hand side of this comparison.
- *
- * @param rhs The new right hand side.
- * @return This, to allow chaining.
- */
- public ComparisonNode setRHS(ExpressionNode rhs) {
- this.rhs = rhs;
- return this;
- }
-
- public OrderingSpecification getOrdering(IdNode lhs, LiteralNode rhs, String operator, int order) {
- if (lhs.getWidthBits() == -1 || lhs.getDivisionBits() == -1 || !(rhs.getValue() instanceof Long)) {
- return null;
- }
-
- if (operator.equals("==") || operator.equals("=")) {
- return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
- }
-
- if (order == OrderingSpecification.ASCENDING) {
- if ((operator.equals("<") || operator.equals("<="))) {
- return new OrderingSpecification(order, 0, lhs.getWidthBits(), lhs.getDivisionBits());
- }
- if (operator.equals(">")) {
- return new OrderingSpecification(order, (Long)rhs.getValue() + 1, lhs.getWidthBits(), lhs.getDivisionBits());
- }
- if (operator.equals(">=")) {
- return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
- }
- } else {
- if (operator.equals("<")) {
- return new OrderingSpecification(order, (Long)rhs.getValue() - 1, lhs.getWidthBits(), lhs.getDivisionBits());
- }
- if (operator.equals("<=")) {
- return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
- }
- }
- return null;
- }
-
- public OrderingSpecification getOrdering(int order) {
- if (lhs instanceof IdNode && rhs instanceof LiteralNode) {
- return getOrdering((IdNode)lhs, (LiteralNode)rhs, operator, order);
- } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) {
- return getOrdering((IdNode)rhs, (LiteralNode)rhs, operator, order);
- }
-
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- if (operator.equals("==") || operator.equals("=")) {
- if (lhs instanceof IdNode && rhs instanceof LiteralNode) {
- return compare(factory, (IdNode)lhs, (LiteralNode)rhs, operator);
- } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) {
- return compare(factory, (IdNode)rhs, (LiteralNode)lhs, operator);
- } else if (lhs instanceof SearchColumnNode && rhs instanceof LiteralNode) {
- return compare(factory, (SearchColumnNode)lhs, (LiteralNode)rhs);
- } else if (rhs instanceof SearchColumnNode && lhs instanceof LiteralNode) {
- return compare(factory, (SearchColumnNode)rhs, (LiteralNode)lhs);
- }
- }
- return null;
- }
-
- /**
- * Compares a search column node with a literal node.
- *
- * @param factory The bucket id factory used.
- * @param node The search column node.
- * @param literal The literal node to compare to.
- * @return The bucket set containing the buckets covered.
- */
- private BucketSet compare(BucketIdFactory factory, SearchColumnNode node, LiteralNode literal) {
- Object value = literal.getValue();
- int bucketCount = (int) Math.pow(2, 16);
- if (value instanceof Long) {
- BucketSet ret = new BucketSet();
- for (int i = 0; i < bucketCount; i++) {
- BucketId id = new BucketId(16, i);
- if ((Long)value == node.getDistribution().getColumn(id)) {
- ret.add(new BucketId(16, i));
- }
- }
- return ret;
- }
- return null;
- }
-
- private BucketSet compare(BucketIdFactory factory, IdNode id, LiteralNode literal, String operator) {
- String field = id.getField();
- Object value = literal.getValue();
- if (field == null) {
- if (value instanceof String) {
- String name = (String)value;
- if ((operator.equals("=") && name.contains("*")) ||
- (operator.equals("=~") && ((name.contains("*") || name.contains("?")))))
- {
- return null; // no idea
- }
- return new BucketSet(factory.getBucketId(new DocumentId(name)));
- }
- } else if (field.equalsIgnoreCase("user")) {
- if (value instanceof Long) {
- return new BucketSet(new BucketId(factory.getLocationBitCount(), (Long)value));
- }
- } else if (field.equalsIgnoreCase("group")) {
- if (value instanceof String) {
- String name = (String)value;
- if ((operator.equals("=") && name.contains("*")) ||
- (operator.equals("=~") && ((name.contains("*") || name.contains("?")))))
- {
- return null; // no idea
- }
- return new BucketSet(new BucketId(factory.getLocationBitCount(), new GroupDocIdString("", name, "").getLocation()));
- }
- } else if (field.equalsIgnoreCase("bucket")) {
- if (value instanceof Long) {
- return new BucketSet(new BucketId((Long)value));
- }
- }
- return null;
- }
-
- // Inherit doc from Node.
- public Object evaluate(Context context) {
- Object oLeft = lhs.evaluate(context);
- Object oRight = rhs.evaluate(context);
- if (oLeft == null && oRight == null) {
- return new ResultList(Result.TRUE);
- }
- if (oLeft == Result.INVALID || oRight == Result.INVALID) {
- return new ResultList(Result.INVALID);
- }
- if (oLeft instanceof AttributeNode.VariableValueList && oRight instanceof AttributeNode.VariableValueList) {
- if (operator.equals("==")) {
- return evaluateListsTrue((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight);
- } else if (operator.equals("!=")) {
- return evaluateListsFalse((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight);
- } else {
- return new ResultList(Result.INVALID);
- }
- } else if (oLeft instanceof AttributeNode.VariableValueList) {
- return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight);
- } else if (oRight instanceof AttributeNode.VariableValueList) {
- return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft);
- }
- return new ResultList(evaluateBool(oLeft, oRight));
- }
-
- public ResultList evaluateListsTrue(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) {
- if (lhs.size() != rhs.size()) {
- return new ResultList(Result.FALSE);
- }
-
- for (int i = 0; i < lhs.size(); i++) {
- if (!lhs.get(i).getVariables().equals(rhs.get(i).getVariables())) {
- return new ResultList(Result.FALSE);
- }
-
- if (evaluateEquals(lhs.get(i).getValue(), rhs.get(i).getValue()) == Result.FALSE) {
- return new ResultList(Result.FALSE);
- }
- }
-
- return new ResultList(Result.TRUE);
- }
-
- public ResultList evaluateListsFalse(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) {
- ResultList lst = evaluateListsTrue(lhs, rhs);
- if (lst.toResult() == Result.TRUE) {
- return new ResultList(Result.FALSE);
- } else if (lst.toResult() == Result.FALSE) {
- return new ResultList(Result.TRUE);
- } else {
- return lst;
- }
- }
-
- public ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) {
- if (rhs == null && lhs == null) {
- return new ResultList(Result.TRUE);
- }
-
- if (rhs == null || lhs == null) {
- return new ResultList(Result.FALSE);
- }
-
- ResultList retVal = new ResultList();
- for (int i = 0; i < lhs.size(); i++) {
- Result result = evaluateBool(lhs.get(i).getValue(), rhs);
- retVal.add((FieldPathIteratorHandler.VariableMap)lhs.get(i).getVariables().clone(), result);
- }
-
- return retVal;
- }
-
- /**
- * Evaluate this expression on two operands, given that they are not invalid.
- *
- * @param lhs Left hand side of operation.
- * @param rhs Right hand side of operation.
- * @return The evaluation result.
- */
- private Result evaluateBool(Object lhs, Object rhs) {
- if (operator.equals("==")) {
- return evaluateEquals(lhs, rhs);
- } else if (operator.equals("!=")) {
- return Result.invert(evaluateEquals(lhs, rhs));
- } else if (operator.equals("<") || operator.equals("<=") ||
- operator.equals(">") || operator.equals(">=")) {
- return evaluateNumber(lhs, rhs);
- } else if (operator.equals("=~") || operator.equals("=")) {
- return evaluateString(lhs, rhs);
- }
- throw new IllegalStateException("Comparison operator '" + operator + "' is not supported.");
- }
-
- /**
- * Compare two operands for equality.
- *
- * @param lhs Left hand side of operation.
- * @param rhs Right hand side of operation.
- * @return Wether or not the two operands are equal.
- */
- private Result evaluateEquals(Object lhs, Object rhs) {
- if (lhs == null || rhs == null) {
- return Result.toResult(lhs == rhs);
- }
-
- double a = getAsNumber(lhs);
- double b = getAsNumber(rhs);
- if (Double.isNaN(a) || Double.isNaN(b)) {
- return Result.toResult(lhs.toString().equals(rhs.toString()));
- }
- return Result.toResult(a == b); // Ugh, comparing doubles? Should be converted to long value perhaps...
- }
-
- private double getAsNumber(Object value) {
- if (value instanceof Number) {
- return ((Number)value).doubleValue();
- } else if (value instanceof NumericFieldValue) {
- return getAsNumber(((NumericFieldValue)value).getNumber());
- } else {
- return Double.NaN; //new IllegalStateException("Term '" + value + "' (" + value.getClass() + ") does not evaluate to a number.");
- }
- }
-
- /**
- * Evalutes the value of this term over a document, given that both operands must be numbers.
- *
- * @param lhs Left hand side of operation.
- * @param rhs Right hand side of operation.
- * @return The evaluation result.
- */
- private Result evaluateNumber(Object lhs, Object rhs) {
- double a = getAsNumber(lhs);
- double b = getAsNumber(rhs);
- if (Double.isNaN(a) || Double.isNaN(b)) {
- return Result.INVALID;
- }
- if (operator.equals("<")) {
- return Result.toResult(a < b);
- } else if (operator.equals("<=")) {
- return Result.toResult(a <= b);
- } else if (operator.equals(">")) {
- return Result.toResult(a > b);
- } else {
- return Result.toResult(a >= b);
- }
- }
-
- /**
- * Evalutes the value of this term over a document, given that both operands must be strings.
- *
- * @param lhs Left hand side of operation.
- * @param rhs Right hand side of operation.
- * @return The evaluation result.
- */
- private Result evaluateString(Object lhs, Object rhs) {
- String left = "" + lhs; // Allows null objects to evaluate to string.
- String right = "" + rhs;
- if (operator.equals("=~")) {
- return Result.toResult(Pattern.compile(right).matcher(left).find());
- } else {
- return Result.toResult(Pattern.compile(globToRegex(right)).matcher(left).find());
- }
- }
-
- /**
- * Converts a glob pattern to a corresponding regular expression string.
- *
- * @param glob The glob pattern.
- * @return The regex string.
- */
- private String globToRegex(String glob) {
- StringBuilder ret = new StringBuilder();
- ret.append("^");
- for (int i = 0; i < glob.length(); i++) {
- ret.append(globToRegex(glob.charAt(i)));
- }
- ret.append("$");
-
- return ret.toString();
- }
-
- /**
- * Converts a single character in a glob expression to the corresponding regular expression string.
- *
- * @param glob The glob character.
- * @return The regex string.
- */
- private String globToRegex(char glob) {
- switch (glob) {
- case'*':
- return ".*";
- case'?':
- return ".";
- case'^':
- case'$':
- case'|':
- case'{':
- case'}':
- case'(':
- case')':
- case'[':
- case']':
- case'\\':
- case'+':
- case'.':
- return "\\" + glob;
- default:
- return "" + glob;
- }
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- // Inherit doc from Object.
- @Override
- public String toString() {
- return lhs + " " + operator + " " + rhs;
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.DocumentId; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.document.idstring.GroupDocIdString; +import com.yahoo.document.select.*; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ComparisonNode implements ExpressionNode { + + // The left- and right-hand-side of this comparison. + private ExpressionNode lhs, rhs; + + // The operator string for this. + private String operator; + + /** + * Constructs a new comparison node. + * + * @param lhs The left-hand-side of the comparison. + * @param operator The comparison operator. + * @param rhs The right-hand-side of the comparison. + */ + public ComparisonNode(ExpressionNode lhs, String operator, ExpressionNode rhs) { + this.lhs = lhs; + this.operator = operator; + this.rhs = rhs; + } + + /** + * Returns the left hand side of this comparison. + * + * @return The left hand side expression. + */ + public ExpressionNode getLHS() { + return lhs; + } + + /** + * Sets the left hand side of this comparison. + * + * @param lhs The new left hand side. + * @return This, to allow chaining. + */ + public ComparisonNode setLHS(ExpressionNode lhs) { + this.lhs = lhs; + return this; + } + + /** + * Returns the comparison operator of this. + * + * @return The operator. + */ + public String getOperator() { + return operator; + } + + /** + * Sets the comparison operator of this. + * + * @param operator The operator string. + * @return This, to allow chaining. + */ + public ComparisonNode setOperator(String operator) { + this.operator = operator; + return this; + } + + /** + * Returns the right hand side of this comparison. + * + * @return The right hand side expression. + */ + public ExpressionNode getRHS() { + return rhs; + } + + /** + * Sets the right hand side of this comparison. + * + * @param rhs The new right hand side. + * @return This, to allow chaining. + */ + public ComparisonNode setRHS(ExpressionNode rhs) { + this.rhs = rhs; + return this; + } + + public OrderingSpecification getOrdering(IdNode lhs, LiteralNode rhs, String operator, int order) { + if (lhs.getWidthBits() == -1 || lhs.getDivisionBits() == -1 || !(rhs.getValue() instanceof Long)) { + return null; + } + + if (operator.equals("==") || operator.equals("=")) { + return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits()); + } + + if (order == OrderingSpecification.ASCENDING) { + if ((operator.equals("<") || operator.equals("<="))) { + return new OrderingSpecification(order, 0, lhs.getWidthBits(), lhs.getDivisionBits()); + } + if (operator.equals(">")) { + return new OrderingSpecification(order, (Long)rhs.getValue() + 1, lhs.getWidthBits(), lhs.getDivisionBits()); + } + if (operator.equals(">=")) { + return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits()); + } + } else { + if (operator.equals("<")) { + return new OrderingSpecification(order, (Long)rhs.getValue() - 1, lhs.getWidthBits(), lhs.getDivisionBits()); + } + if (operator.equals("<=")) { + return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits()); + } + } + return null; + } + + public OrderingSpecification getOrdering(int order) { + if (lhs instanceof IdNode && rhs instanceof LiteralNode) { + return getOrdering((IdNode)lhs, (LiteralNode)rhs, operator, order); + } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) { + return getOrdering((IdNode)rhs, (LiteralNode)rhs, operator, order); + } + + return null; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + if (operator.equals("==") || operator.equals("=")) { + if (lhs instanceof IdNode && rhs instanceof LiteralNode) { + return compare(factory, (IdNode)lhs, (LiteralNode)rhs, operator); + } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) { + return compare(factory, (IdNode)rhs, (LiteralNode)lhs, operator); + } else if (lhs instanceof SearchColumnNode && rhs instanceof LiteralNode) { + return compare(factory, (SearchColumnNode)lhs, (LiteralNode)rhs); + } else if (rhs instanceof SearchColumnNode && lhs instanceof LiteralNode) { + return compare(factory, (SearchColumnNode)rhs, (LiteralNode)lhs); + } + } + return null; + } + + /** + * Compares a search column node with a literal node. + * + * @param factory The bucket id factory used. + * @param node The search column node. + * @param literal The literal node to compare to. + * @return The bucket set containing the buckets covered. + */ + private BucketSet compare(BucketIdFactory factory, SearchColumnNode node, LiteralNode literal) { + Object value = literal.getValue(); + int bucketCount = (int) Math.pow(2, 16); + if (value instanceof Long) { + BucketSet ret = new BucketSet(); + for (int i = 0; i < bucketCount; i++) { + BucketId id = new BucketId(16, i); + if ((Long)value == node.getDistribution().getColumn(id)) { + ret.add(new BucketId(16, i)); + } + } + return ret; + } + return null; + } + + private BucketSet compare(BucketIdFactory factory, IdNode id, LiteralNode literal, String operator) { + String field = id.getField(); + Object value = literal.getValue(); + if (field == null) { + if (value instanceof String) { + String name = (String)value; + if ((operator.equals("=") && name.contains("*")) || + (operator.equals("=~") && ((name.contains("*") || name.contains("?"))))) + { + return null; // no idea + } + return new BucketSet(factory.getBucketId(new DocumentId(name))); + } + } else if (field.equalsIgnoreCase("user")) { + if (value instanceof Long) { + return new BucketSet(new BucketId(factory.getLocationBitCount(), (Long)value)); + } + } else if (field.equalsIgnoreCase("group")) { + if (value instanceof String) { + String name = (String)value; + if ((operator.equals("=") && name.contains("*")) || + (operator.equals("=~") && ((name.contains("*") || name.contains("?"))))) + { + return null; // no idea + } + return new BucketSet(new BucketId(factory.getLocationBitCount(), new GroupDocIdString("", name, "").getLocation())); + } + } else if (field.equalsIgnoreCase("bucket")) { + if (value instanceof Long) { + return new BucketSet(new BucketId((Long)value)); + } + } + return null; + } + + // Inherit doc from Node. + public Object evaluate(Context context) { + Object oLeft = lhs.evaluate(context); + Object oRight = rhs.evaluate(context); + if (oLeft == null && oRight == null) { + return new ResultList(Result.TRUE); + } + if (oLeft == Result.INVALID || oRight == Result.INVALID) { + return new ResultList(Result.INVALID); + } + if (oLeft instanceof AttributeNode.VariableValueList && oRight instanceof AttributeNode.VariableValueList) { + if (operator.equals("==")) { + return evaluateListsTrue((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight); + } else if (operator.equals("!=")) { + return evaluateListsFalse((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight); + } else { + return new ResultList(Result.INVALID); + } + } else if (oLeft instanceof AttributeNode.VariableValueList) { + return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight); + } else if (oRight instanceof AttributeNode.VariableValueList) { + return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft); + } + return new ResultList(evaluateBool(oLeft, oRight)); + } + + public ResultList evaluateListsTrue(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) { + if (lhs.size() != rhs.size()) { + return new ResultList(Result.FALSE); + } + + for (int i = 0; i < lhs.size(); i++) { + if (!lhs.get(i).getVariables().equals(rhs.get(i).getVariables())) { + return new ResultList(Result.FALSE); + } + + if (evaluateEquals(lhs.get(i).getValue(), rhs.get(i).getValue()) == Result.FALSE) { + return new ResultList(Result.FALSE); + } + } + + return new ResultList(Result.TRUE); + } + + public ResultList evaluateListsFalse(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) { + ResultList lst = evaluateListsTrue(lhs, rhs); + if (lst.toResult() == Result.TRUE) { + return new ResultList(Result.FALSE); + } else if (lst.toResult() == Result.FALSE) { + return new ResultList(Result.TRUE); + } else { + return lst; + } + } + + public ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) { + if (rhs == null && lhs == null) { + return new ResultList(Result.TRUE); + } + + if (rhs == null || lhs == null) { + return new ResultList(Result.FALSE); + } + + ResultList retVal = new ResultList(); + for (int i = 0; i < lhs.size(); i++) { + Result result = evaluateBool(lhs.get(i).getValue(), rhs); + retVal.add((FieldPathIteratorHandler.VariableMap)lhs.get(i).getVariables().clone(), result); + } + + return retVal; + } + + /** + * Evaluate this expression on two operands, given that they are not invalid. + * + * @param lhs Left hand side of operation. + * @param rhs Right hand side of operation. + * @return The evaluation result. + */ + private Result evaluateBool(Object lhs, Object rhs) { + if (operator.equals("==")) { + return evaluateEquals(lhs, rhs); + } else if (operator.equals("!=")) { + return Result.invert(evaluateEquals(lhs, rhs)); + } else if (operator.equals("<") || operator.equals("<=") || + operator.equals(">") || operator.equals(">=")) { + return evaluateNumber(lhs, rhs); + } else if (operator.equals("=~") || operator.equals("=")) { + return evaluateString(lhs, rhs); + } + throw new IllegalStateException("Comparison operator '" + operator + "' is not supported."); + } + + /** + * Compare two operands for equality. + * + * @param lhs Left hand side of operation. + * @param rhs Right hand side of operation. + * @return Wether or not the two operands are equal. + */ + private Result evaluateEquals(Object lhs, Object rhs) { + if (lhs == null || rhs == null) { + return Result.toResult(lhs == rhs); + } + + double a = getAsNumber(lhs); + double b = getAsNumber(rhs); + if (Double.isNaN(a) || Double.isNaN(b)) { + return Result.toResult(lhs.toString().equals(rhs.toString())); + } + return Result.toResult(a == b); // Ugh, comparing doubles? Should be converted to long value perhaps... + } + + private double getAsNumber(Object value) { + if (value instanceof Number) { + return ((Number)value).doubleValue(); + } else if (value instanceof NumericFieldValue) { + return getAsNumber(((NumericFieldValue)value).getNumber()); + } else { + return Double.NaN; //new IllegalStateException("Term '" + value + "' (" + value.getClass() + ") does not evaluate to a number."); + } + } + + /** + * Evalutes the value of this term over a document, given that both operands must be numbers. + * + * @param lhs Left hand side of operation. + * @param rhs Right hand side of operation. + * @return The evaluation result. + */ + private Result evaluateNumber(Object lhs, Object rhs) { + double a = getAsNumber(lhs); + double b = getAsNumber(rhs); + if (Double.isNaN(a) || Double.isNaN(b)) { + return Result.INVALID; + } + if (operator.equals("<")) { + return Result.toResult(a < b); + } else if (operator.equals("<=")) { + return Result.toResult(a <= b); + } else if (operator.equals(">")) { + return Result.toResult(a > b); + } else { + return Result.toResult(a >= b); + } + } + + /** + * Evalutes the value of this term over a document, given that both operands must be strings. + * + * @param lhs Left hand side of operation. + * @param rhs Right hand side of operation. + * @return The evaluation result. + */ + private Result evaluateString(Object lhs, Object rhs) { + String left = "" + lhs; // Allows null objects to evaluate to string. + String right = "" + rhs; + if (operator.equals("=~")) { + return Result.toResult(Pattern.compile(right).matcher(left).find()); + } else { + return Result.toResult(Pattern.compile(globToRegex(right)).matcher(left).find()); + } + } + + /** + * Converts a glob pattern to a corresponding regular expression string. + * + * @param glob The glob pattern. + * @return The regex string. + */ + private String globToRegex(String glob) { + StringBuilder ret = new StringBuilder(); + ret.append("^"); + for (int i = 0; i < glob.length(); i++) { + ret.append(globToRegex(glob.charAt(i))); + } + ret.append("$"); + + return ret.toString(); + } + + /** + * Converts a single character in a glob expression to the corresponding regular expression string. + * + * @param glob The glob character. + * @return The regex string. + */ + private String globToRegex(char glob) { + switch (glob) { + case'*': + return ".*"; + case'?': + return "."; + case'^': + case'$': + case'|': + case'{': + case'}': + case'(': + case')': + case'[': + case']': + case'\\': + case'+': + case'.': + return "\\" + glob; + default: + return "" + glob; + } + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + // Inherit doc from Object. + @Override + public String toString() { + return lhs + " " + operator + " " + rhs; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java index c071e6674aa..2b095db16c9 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java @@ -1,65 +1,65 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.*;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.Visitor;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class DocumentNode implements ExpressionNode {
-
- private String type;
-
- public DocumentNode(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
-
- public DocumentNode setType(String type) {
- this.type = type;
- return this;
- }
-
- @Override
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- @Override
- public Object evaluate(Context context) {
- return evaluate(context.getDocumentOperation());
- }
-
- public Object evaluate(DocumentOperation op) {
- DocumentType doct;
- if (op instanceof DocumentPut) {
- doct = ((DocumentPut)op).getDocument().getDataType();
- } else if (op instanceof DocumentUpdate) {
- doct = ((DocumentUpdate)op).getDocumentType();
- } else {
- throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported.");
- }
- return doct.isA(this.type) ? op : Boolean.FALSE;
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- return type;
- }
-
- @Override
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.*; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.Visitor; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentNode implements ExpressionNode { + + private String type; + + public DocumentNode(String type) { + this.type = type; + } + + public String getType() { + return type; + } + + public DocumentNode setType(String type) { + this.type = type; + return this; + } + + @Override + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + @Override + public Object evaluate(Context context) { + return evaluate(context.getDocumentOperation()); + } + + public Object evaluate(DocumentOperation op) { + DocumentType doct; + if (op instanceof DocumentPut) { + doct = ((DocumentPut)op).getDocument().getDataType(); + } else if (op instanceof DocumentUpdate) { + doct = ((DocumentUpdate)op).getDocumentType(); + } else { + throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported."); + } + return doct.isA(this.type) ? op : Boolean.FALSE; + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return type; + } + + @Override + public OrderingSpecification getOrdering(int order) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java b/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java index 13b20695cab..12192af7607 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java @@ -1,51 +1,51 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.Visitor;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class EmbracedNode implements ExpressionNode {
-
- private ExpressionNode node;
-
- public EmbracedNode(ExpressionNode node) {
- this.node = node;
- }
-
- public ExpressionNode getNode() {
- return node;
- }
-
- public EmbracedNode setNode(ExpressionNode node) {
- this.node = node;
- return this;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return node.getBucketSet(factory);
- }
-
- public Object evaluate(Context context) {
- return node.evaluate(context);
- }
-
- @Override
- public String toString() {
- return "(" + node + ")";
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.Visitor; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class EmbracedNode implements ExpressionNode { + + private ExpressionNode node; + + public EmbracedNode(ExpressionNode node) { + this.node = node; + } + + public ExpressionNode getNode() { + return node; + } + + public EmbracedNode setNode(ExpressionNode node) { + this.node = node; + return this; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return node.getBucketSet(factory); + } + + public Object evaluate(Context context) { + return node.evaluate(context); + } + + @Override + public String toString() { + return "(" + node + ")"; + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + public OrderingSpecification getOrdering(int order) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java b/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java index ba51ec840d4..d84259f2643 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java @@ -1,48 +1,48 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.Visitor;
-
-/**
- * This is the interface of all expression nodes. It declares the methods requires by all expression nodes to maintain
- * a working document selector language.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public interface ExpressionNode {
-
- /**
- * Evaluate the content of this node based on document object, and return that value.
- *
- * @param doc The document to evaluate over.
- * @return The value of this.
- */
- public Object evaluate(Context doc);
-
- /**
- * Returns the set of bucket ids covered by this node.
- *
- * @param factory The factory used by the current application.
- */
- public BucketSet getBucketSet(BucketIdFactory factory);
-
- /**
- * If this document selection implies a specific ordering (using the orderdoc scheme),
- * return that specification.
- *
- * @param order The order in which we are looking to traverse the ordering (ASCENDING or DESCENDING)
- */
- public OrderingSpecification getOrdering(int order);
-
- /**
- * Perform visitation of this node.
- *
- * @param visitor The visitor that wishes to visit the node.
- */
- public void accept(Visitor visitor);
-
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.Visitor; + +/** + * This is the interface of all expression nodes. It declares the methods requires by all expression nodes to maintain + * a working document selector language. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface ExpressionNode { + + /** + * Evaluate the content of this node based on document object, and return that value. + * + * @param doc The document to evaluate over. + * @return The value of this. + */ + public Object evaluate(Context doc); + + /** + * Returns the set of bucket ids covered by this node. + * + * @param factory The factory used by the current application. + */ + public BucketSet getBucketSet(BucketIdFactory factory); + + /** + * If this document selection implies a specific ordering (using the orderdoc scheme), + * return that specification. + * + * @param order The order in which we are looking to traverse the ordering (ASCENDING or DESCENDING) + */ + public OrderingSpecification getOrdering(int order); + + /** + * Perform visitation of this node. + * + * @param visitor The visitor that wishes to visit the node. + */ + public void accept(Visitor visitor); + +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/IdNode.java b/document/src/main/java/com/yahoo/document/select/rule/IdNode.java index f12ecbb752b..75b4890af80 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/IdNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/IdNode.java @@ -1,108 +1,108 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.*;
-import com.yahoo.document.idstring.*;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class IdNode implements ExpressionNode {
-
- private String field;
- private short widthBits = -1;
- private short divisionBits = -1;
-
- public IdNode() {
- // empty
- }
-
- public String getField() {
- return field;
- }
-
- public IdNode setField(String field) {
- this.field = field;
- return this;
- }
-
- public IdNode setWidthBits(short widthBits) {
- this.widthBits = widthBits;
- return this;
- }
-
- public IdNode setDivisionBits(short divisionBits) {
- this.divisionBits = divisionBits;
- return this;
- }
-
- public short getWidthBits() {
- return widthBits;
- }
-
- public short getDivisionBits() {
- return divisionBits;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- public OrderingSpecification getOrdering(int ordering) {
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public Object evaluate(Context context) {
- DocumentId id = context.getDocumentOperation().getId();
- if (id == null) {
- throw new IllegalStateException("Document has no identifier.");
- }
- if (field == null) {
- return id.toString();
- } else if (field.equalsIgnoreCase("scheme")) {
- return id.getScheme().getType().toString();
- } else if (field.equalsIgnoreCase("namespace")) {
- return id.getScheme().getNamespace();
- } else if (field.equalsIgnoreCase("specific")) {
- return id.getScheme().getNamespaceSpecific();
- } else if (field.equalsIgnoreCase("group")) {
- if (id.getScheme().hasGroup()) {
- return id.getScheme().getGroup();
- }
- throw new IllegalStateException("Group identifier is null.");
- } else if (field.equalsIgnoreCase("user")) {
- if (id.getScheme().hasNumber()) {
- return id.getScheme().getNumber();
- }
- throw new IllegalStateException("User identifier is null.");
- } else if (field.equalsIgnoreCase("type")) {
- if (id.getScheme().hasDocType()) {
- return id.getScheme().getDocType();
- }
- throw new IllegalStateException("Document id doesn't have doc type.");
- } else if (field.equalsIgnoreCase("order")) {
- if (id.getScheme() instanceof OrderDocIdString) {
- OrderDocIdString ods = (OrderDocIdString)id.getScheme();
- if (ods.getWidthBits() == widthBits && ods.getDivisionBits() == divisionBits) {
- return ods.getOrdering();
- }
- }
- } else{
- throw new IllegalStateException("Identifier field '" + field + "' is not supported.");
- }
- return null;
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- return "id" + (field != null ? "." + field : "") + (widthBits != -1 ? "(" + widthBits + "," + divisionBits + ")" : "");
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.*; +import com.yahoo.document.idstring.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IdNode implements ExpressionNode { + + private String field; + private short widthBits = -1; + private short divisionBits = -1; + + public IdNode() { + // empty + } + + public String getField() { + return field; + } + + public IdNode setField(String field) { + this.field = field; + return this; + } + + public IdNode setWidthBits(short widthBits) { + this.widthBits = widthBits; + return this; + } + + public IdNode setDivisionBits(short divisionBits) { + this.divisionBits = divisionBits; + return this; + } + + public short getWidthBits() { + return widthBits; + } + + public short getDivisionBits() { + return divisionBits; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + public OrderingSpecification getOrdering(int ordering) { + return null; + } + + // Inherit doc from ExpressionNode. + public Object evaluate(Context context) { + DocumentId id = context.getDocumentOperation().getId(); + if (id == null) { + throw new IllegalStateException("Document has no identifier."); + } + if (field == null) { + return id.toString(); + } else if (field.equalsIgnoreCase("scheme")) { + return id.getScheme().getType().toString(); + } else if (field.equalsIgnoreCase("namespace")) { + return id.getScheme().getNamespace(); + } else if (field.equalsIgnoreCase("specific")) { + return id.getScheme().getNamespaceSpecific(); + } else if (field.equalsIgnoreCase("group")) { + if (id.getScheme().hasGroup()) { + return id.getScheme().getGroup(); + } + throw new IllegalStateException("Group identifier is null."); + } else if (field.equalsIgnoreCase("user")) { + if (id.getScheme().hasNumber()) { + return id.getScheme().getNumber(); + } + throw new IllegalStateException("User identifier is null."); + } else if (field.equalsIgnoreCase("type")) { + if (id.getScheme().hasDocType()) { + return id.getScheme().getDocType(); + } + throw new IllegalStateException("Document id doesn't have doc type."); + } else if (field.equalsIgnoreCase("order")) { + if (id.getScheme() instanceof OrderDocIdString) { + OrderDocIdString ods = (OrderDocIdString)id.getScheme(); + if (ods.getWidthBits() == widthBits && ods.getDivisionBits() == divisionBits) { + return ods.getOrdering(); + } + } + } else{ + throw new IllegalStateException("Identifier field '" + field + "' is not supported."); + } + return null; + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return "id" + (field != null ? "." + field : "") + (widthBits != -1 ? "(" + widthBits + "," + divisionBits + ")" : ""); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java b/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java index ae0d640d471..702026b59d3 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java @@ -1,61 +1,61 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.Visitor;
-import com.yahoo.document.select.parser.SelectParserUtils;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class LiteralNode implements ExpressionNode {
-
- private Object value;
-
- public LiteralNode(Object value) {
- this.value = value;
- }
-
- public Object getValue() {
- return value;
- }
-
- public LiteralNode setValue(Object value) {
- this.value = value;
- return this;
- }
-
- @Override
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- @Override
- public Object evaluate(Context context) {
- return value;
- }
-
- @Override
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- if (value == null) {
- return "null";
- } else if (value instanceof String) {
- return SelectParserUtils.quote((String)value, '"');
- } else {
- return value.toString();
- }
- }
-
- @Override
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.Visitor; +import com.yahoo.document.select.parser.SelectParserUtils; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LiteralNode implements ExpressionNode { + + private Object value; + + public LiteralNode(Object value) { + this.value = value; + } + + public Object getValue() { + return value; + } + + public LiteralNode setValue(Object value) { + this.value = value; + return this; + } + + @Override + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + @Override + public Object evaluate(Context context) { + return value; + } + + @Override + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + if (value == null) { + return "null"; + } else if (value instanceof String) { + return SelectParserUtils.quote((String)value, '"'); + } else { + return value.toString(); + } + } + + @Override + public OrderingSpecification getOrdering(int order) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java index 145c655fae2..edff3a0adaa 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java @@ -1,316 +1,316 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.ResultList;
-import com.yahoo.document.select.Visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Stack;
-
-/**
- * This class defines a logical expression of nodes. This implementation uses a stack to evaluate its content as to
- * avoid deep recursions when building the parse tree.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class LogicNode implements ExpressionNode {
-
- // A no-op value is defined for completeness.
- public static final int NOP = 0;
-
- // The OR operator has lower precedence than AND.
- public static final int OR = 1;
-
- // The AND operator has the highest precedence.
- public static final int AND = 2;
-
- // The items contained in this.
- private final List<NodeItem> items = new ArrayList<NodeItem>();
-
- /**
- * Construct an empty logic expression.
- */
- public LogicNode() {
- // empty
- }
-
- public List<NodeItem> getItems() {
- return items;
- }
-
- /**
- * Adds an (operator, node) pair to this expression.
- *
- * @param operator The operator that combines the previous with the node given.
- * @param node The node to add to this.
- * @return This, to allow chaining.
- */
- public LogicNode add(String operator, ExpressionNode node) {
- items.add(new LogicNode.NodeItem(stringToOperator(operator), node));
- return this;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- Stack<BucketItem> buf = new Stack<>();
- for (NodeItem item : items) {
- if (!buf.isEmpty()) {
- while (buf.peek().operator > item.operator) {
- combineBuckets(buf);
- }
- }
- buf.push(new BucketItem(item.operator, item.node.getBucketSet(factory)));
- }
- while (buf.size() > 1) {
- combineBuckets(buf);
- }
- return buf.pop().buckets;
- }
-
- public OrderingSpecification getOrdering(int order) {
- Stack<OrderingItem> buf = new Stack<>();
- for (NodeItem item : items) {
- if (!buf.isEmpty()) {
- while (buf.peek().operator > item.operator) {
- pickOrdering(buf);
- }
- }
- buf.push(new OrderingItem(item.operator, item.node.getOrdering(order)));
- }
- while (buf.size() > 1) {
- pickOrdering(buf);
- }
- return buf.pop().ordering;
- }
-
- private OrderingSpecification pickOrdering(OrderingSpecification a, OrderingSpecification b, boolean isAnd) {
- if (a.getWidthBits() == b.getWidthBits() && a.getDivisionBits() == b.getDivisionBits() && a.getOrder() == b.getOrder()) {
- if ((a.getOrder() == OrderingSpecification.ASCENDING && isAnd) ||
- (a.getOrder() == OrderingSpecification.DESCENDING && !isAnd)) {
- return new OrderingSpecification(a.getOrder(), Math.max(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits());
- } else {
- return new OrderingSpecification(a.getOrder(), Math.min(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits());
- }
- }
- return null;
- }
-
- private void pickOrdering(Stack<OrderingItem> buf) {
- OrderingItem rhs = buf.pop();
- OrderingItem lhs = buf.pop();
- switch (rhs.operator) {
- case AND:
- if (lhs.ordering == null) {
- lhs.ordering = rhs.ordering;
- } else if (rhs.ordering == null) {
- // empty
- } else {
- lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, true);
- }
- break;
- case OR:
- if (lhs.ordering != null && rhs.ordering != null) {
- lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, false);
- } else {
- lhs.ordering = null;
- }
- break;
- default:
- lhs.ordering = null;
- }
- buf.push(lhs);
- }
-
- /**
- * Combines the top two items of the given stack using the operator of the second.
- *
- * @param buf The stack of bucket items.
- */
- private void combineBuckets(Stack<BucketItem> buf) {
- BucketItem rhs = buf.pop();
- BucketItem lhs = buf.pop();
- switch (rhs.operator) {
- case AND:
- if (lhs.buckets == null) {
- lhs.buckets = rhs.buckets;
- } else if (rhs.buckets == null) {
- // empty
- } else {
- lhs.buckets = lhs.buckets.intersection(rhs.buckets);
- }
- break;
- case OR:
- if (lhs.buckets == null) {
- // empty
- } else if (rhs.buckets == null) {
- lhs.buckets = null;
- } else {
- lhs.buckets = lhs.buckets.union(rhs.buckets);
- }
- break;
- default:
- throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
- }
- buf.push(lhs);
- }
-
- // Inherit doc from ExpressionNode.
- @Override
- public Object evaluate(Context context) {
- Stack<ValueItem> buf = new Stack<>();
- for (NodeItem item : items) {
- if ( ! buf.isEmpty()) {
- while (buf.peek().operator > item.operator) {
- combineValues(buf);
- }
- }
-
- buf.push(new ValueItem(item.operator, ResultList.toResultList(item.node.evaluate(context))));
- }
- while (buf.size() > 1) {
- combineValues(buf);
- }
- return buf.pop().value;
- }
-
- /**
- * Combines the top two items of the given stack using the operator of the second.
- *
- * @param buf The stack of values.
- */
- private void combineValues(Stack<ValueItem> buf) {
- ValueItem rhs = buf.pop();
- ValueItem lhs = buf.pop();
-
- switch (rhs.operator) {
- case AND:
- buf.push(new ValueItem(lhs.operator, lhs.value.combineAND(rhs.value)));
- break;
- case OR:
- buf.push(new ValueItem(lhs.operator, lhs.value.combineOR(rhs.value)));
- break;
- default:
- throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
- }
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- // Inherit doc from Object.
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- for (LogicNode.NodeItem item : items) {
- if (item.operator != NOP) {
- ret.append(" ").append(operatorToString(item.operator)).append(" ");
- }
- ret.append(item.node);
- }
- return ret.toString();
- }
-
- /**
- * Converts the given operator index to a string representation.
- *
- * @param operator The operator index to convert.
- * @return The string representation.
- */
- public String operatorToString(int operator) {
- switch (operator) {
- case NOP:
- return null;
- case OR:
- return "or";
- case AND:
- return "and";
- default:
- throw new IllegalStateException("Logical operator " + operator + " not supported.");
- }
- }
-
- /**
- * Converts the given operator string to a corresponding operator index. This is necessary to perform a stack
- * traversal of logic expression.
- *
- * @param operator The operator to convert.
- * @return The corresponding index.
- */
- private int stringToOperator(String operator) {
- if (operator == null) {
- return NOP;
- } else if (operator.equalsIgnoreCase("or")) {
- return OR;
- } else if (operator.equalsIgnoreCase("and")) {
- return AND;
- } else {
- throw new IllegalStateException("Logical operator '" + operator + "' not supported.");
- }
- }
-
- /**
- * Private class to store results in a stack.
- */
- private final class ValueItem {
- private int operator;
- private ResultList value;
-
- public ValueItem(int operator, ResultList value) {
- this.operator = operator;
- this.value = value;
- }
- }
-
- /**
- * Private class to store bucket sets in a stack.
- */
- private final class BucketItem {
- private int operator;
- private BucketSet buckets;
-
- public BucketItem(int operator, BucketSet buckets) {
- this.operator = operator;
- this.buckets = buckets;
- }
- }
-
- /**
- * Private class to store ordering expressions in a stack.
- */
- private final class OrderingItem {
- private int operator;
- private OrderingSpecification ordering;
-
- public OrderingItem(int operator, OrderingSpecification orderSpec) {
- this.operator = operator;
- this.ordering = orderSpec;
- }
- }
-
- /**
- * Private class to store expression nodes in a stack.
- */
- public final class NodeItem {
- private int operator;
- private ExpressionNode node;
-
- public NodeItem(int operator, ExpressionNode node) {
- this.operator = operator;
- this.node = node;
- }
-
- public int getOperator() {
- return operator;
- }
-
- public ExpressionNode getNode() {
- return node;
- }
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.ResultList; +import com.yahoo.document.select.Visitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +/** + * This class defines a logical expression of nodes. This implementation uses a stack to evaluate its content as to + * avoid deep recursions when building the parse tree. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LogicNode implements ExpressionNode { + + // A no-op value is defined for completeness. + public static final int NOP = 0; + + // The OR operator has lower precedence than AND. + public static final int OR = 1; + + // The AND operator has the highest precedence. + public static final int AND = 2; + + // The items contained in this. + private final List<NodeItem> items = new ArrayList<NodeItem>(); + + /** + * Construct an empty logic expression. + */ + public LogicNode() { + // empty + } + + public List<NodeItem> getItems() { + return items; + } + + /** + * Adds an (operator, node) pair to this expression. + * + * @param operator The operator that combines the previous with the node given. + * @param node The node to add to this. + * @return This, to allow chaining. + */ + public LogicNode add(String operator, ExpressionNode node) { + items.add(new LogicNode.NodeItem(stringToOperator(operator), node)); + return this; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + Stack<BucketItem> buf = new Stack<>(); + for (NodeItem item : items) { + if (!buf.isEmpty()) { + while (buf.peek().operator > item.operator) { + combineBuckets(buf); + } + } + buf.push(new BucketItem(item.operator, item.node.getBucketSet(factory))); + } + while (buf.size() > 1) { + combineBuckets(buf); + } + return buf.pop().buckets; + } + + public OrderingSpecification getOrdering(int order) { + Stack<OrderingItem> buf = new Stack<>(); + for (NodeItem item : items) { + if (!buf.isEmpty()) { + while (buf.peek().operator > item.operator) { + pickOrdering(buf); + } + } + buf.push(new OrderingItem(item.operator, item.node.getOrdering(order))); + } + while (buf.size() > 1) { + pickOrdering(buf); + } + return buf.pop().ordering; + } + + private OrderingSpecification pickOrdering(OrderingSpecification a, OrderingSpecification b, boolean isAnd) { + if (a.getWidthBits() == b.getWidthBits() && a.getDivisionBits() == b.getDivisionBits() && a.getOrder() == b.getOrder()) { + if ((a.getOrder() == OrderingSpecification.ASCENDING && isAnd) || + (a.getOrder() == OrderingSpecification.DESCENDING && !isAnd)) { + return new OrderingSpecification(a.getOrder(), Math.max(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits()); + } else { + return new OrderingSpecification(a.getOrder(), Math.min(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits()); + } + } + return null; + } + + private void pickOrdering(Stack<OrderingItem> buf) { + OrderingItem rhs = buf.pop(); + OrderingItem lhs = buf.pop(); + switch (rhs.operator) { + case AND: + if (lhs.ordering == null) { + lhs.ordering = rhs.ordering; + } else if (rhs.ordering == null) { + // empty + } else { + lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, true); + } + break; + case OR: + if (lhs.ordering != null && rhs.ordering != null) { + lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, false); + } else { + lhs.ordering = null; + } + break; + default: + lhs.ordering = null; + } + buf.push(lhs); + } + + /** + * Combines the top two items of the given stack using the operator of the second. + * + * @param buf The stack of bucket items. + */ + private void combineBuckets(Stack<BucketItem> buf) { + BucketItem rhs = buf.pop(); + BucketItem lhs = buf.pop(); + switch (rhs.operator) { + case AND: + if (lhs.buckets == null) { + lhs.buckets = rhs.buckets; + } else if (rhs.buckets == null) { + // empty + } else { + lhs.buckets = lhs.buckets.intersection(rhs.buckets); + } + break; + case OR: + if (lhs.buckets == null) { + // empty + } else if (rhs.buckets == null) { + lhs.buckets = null; + } else { + lhs.buckets = lhs.buckets.union(rhs.buckets); + } + break; + default: + throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); + } + buf.push(lhs); + } + + // Inherit doc from ExpressionNode. + @Override + public Object evaluate(Context context) { + Stack<ValueItem> buf = new Stack<>(); + for (NodeItem item : items) { + if ( ! buf.isEmpty()) { + while (buf.peek().operator > item.operator) { + combineValues(buf); + } + } + + buf.push(new ValueItem(item.operator, ResultList.toResultList(item.node.evaluate(context)))); + } + while (buf.size() > 1) { + combineValues(buf); + } + return buf.pop().value; + } + + /** + * Combines the top two items of the given stack using the operator of the second. + * + * @param buf The stack of values. + */ + private void combineValues(Stack<ValueItem> buf) { + ValueItem rhs = buf.pop(); + ValueItem lhs = buf.pop(); + + switch (rhs.operator) { + case AND: + buf.push(new ValueItem(lhs.operator, lhs.value.combineAND(rhs.value))); + break; + case OR: + buf.push(new ValueItem(lhs.operator, lhs.value.combineOR(rhs.value))); + break; + default: + throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); + } + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + // Inherit doc from Object. + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + for (LogicNode.NodeItem item : items) { + if (item.operator != NOP) { + ret.append(" ").append(operatorToString(item.operator)).append(" "); + } + ret.append(item.node); + } + return ret.toString(); + } + + /** + * Converts the given operator index to a string representation. + * + * @param operator The operator index to convert. + * @return The string representation. + */ + public String operatorToString(int operator) { + switch (operator) { + case NOP: + return null; + case OR: + return "or"; + case AND: + return "and"; + default: + throw new IllegalStateException("Logical operator " + operator + " not supported."); + } + } + + /** + * Converts the given operator string to a corresponding operator index. This is necessary to perform a stack + * traversal of logic expression. + * + * @param operator The operator to convert. + * @return The corresponding index. + */ + private int stringToOperator(String operator) { + if (operator == null) { + return NOP; + } else if (operator.equalsIgnoreCase("or")) { + return OR; + } else if (operator.equalsIgnoreCase("and")) { + return AND; + } else { + throw new IllegalStateException("Logical operator '" + operator + "' not supported."); + } + } + + /** + * Private class to store results in a stack. + */ + private final class ValueItem { + private int operator; + private ResultList value; + + public ValueItem(int operator, ResultList value) { + this.operator = operator; + this.value = value; + } + } + + /** + * Private class to store bucket sets in a stack. + */ + private final class BucketItem { + private int operator; + private BucketSet buckets; + + public BucketItem(int operator, BucketSet buckets) { + this.operator = operator; + this.buckets = buckets; + } + } + + /** + * Private class to store ordering expressions in a stack. + */ + private final class OrderingItem { + private int operator; + private OrderingSpecification ordering; + + public OrderingItem(int operator, OrderingSpecification orderSpec) { + this.operator = operator; + this.ordering = orderSpec; + } + } + + /** + * Private class to store expression nodes in a stack. + */ + public final class NodeItem { + private int operator; + private ExpressionNode node; + + public NodeItem(int operator, ExpressionNode node) { + this.operator = operator; + this.node = node; + } + + public int getOperator() { + return operator; + } + + public ExpressionNode getNode() { + return node; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java b/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java index 2b85bc3eee6..74e6f0286f8 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java @@ -1,53 +1,53 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.Result;
-import com.yahoo.document.select.Visitor;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class NegationNode implements ExpressionNode {
-
- private ExpressionNode node;
-
- public NegationNode(ExpressionNode node) {
- this.node = node;
- }
-
- public ExpressionNode getNode() {
- return node;
- }
-
- public NegationNode setNode(ExpressionNode node) {
- this.node = node;
- return this;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public Object evaluate(Context context) {
- return Result.invert(Result.toResult(node.evaluate(context)));
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- return "not " + node;
- }
-
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.Result; +import com.yahoo.document.select.Visitor; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NegationNode implements ExpressionNode { + + private ExpressionNode node; + + public NegationNode(ExpressionNode node) { + this.node = node; + } + + public ExpressionNode getNode() { + return node; + } + + public NegationNode setNode(ExpressionNode node) { + this.node = node; + return this; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + // Inherit doc from ExpressionNode. + public Object evaluate(Context context) { + return Result.invert(Result.toResult(node.evaluate(context))); + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return "not " + node; + } + + public OrderingSpecification getOrdering(int order) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/NowNode.java b/document/src/main/java/com/yahoo/document/select/rule/NowNode.java index 600dbe536e4..8941f6b2808 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/NowNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/NowNode.java @@ -1,34 +1,34 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.*;
-
-/**
- * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
- */
-public class NowNode implements ExpressionNode {
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public Object evaluate(Context context) {
- Object ret = System.currentTimeMillis() / 1000;
- return ret;
- }
-
- @Override
- public String toString() {
- return "now()";
- }
-
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-}
+package com.yahoo.document.select.rule; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.*; + +/** + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ +public class NowNode implements ExpressionNode { + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + // Inherit doc from ExpressionNode. + public Object evaluate(Context context) { + Object ret = System.currentTimeMillis() / 1000; + return ret; + } + + @Override + public String toString() { + return "now()"; + } + + public OrderingSpecification getOrdering(int order) { + return null; + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java b/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java index c63197ae3a4..03c8b273a77 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java @@ -1,56 +1,56 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketDistribution;
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.*;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class SearchColumnNode implements ExpressionNode {
-
- private int field;
- private BucketIdFactory factory = new BucketIdFactory(); // why is this not an abstract class?
- private BucketDistribution distribution;
-
- public SearchColumnNode() {
- setField(0);
- }
-
- public int getField() {
- return field;
- }
-
- public SearchColumnNode setField(int field) {
- distribution = new BucketDistribution(this.field = field, 16);
- return this;
- }
-
- public BucketDistribution getDistribution() {
- return distribution;
- }
-
- // Inherit doc from ExpressionNode.
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- // Inherit doc from ExpressionNode.
- public Object evaluate(Context context) {
- return distribution.getColumn(factory.getBucketId(context.getDocumentOperation().getId()));
- }
-
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- return "searchcolumn." + field;
- }
-
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketDistribution; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SearchColumnNode implements ExpressionNode { + + private int field; + private BucketIdFactory factory = new BucketIdFactory(); // why is this not an abstract class? + private BucketDistribution distribution; + + public SearchColumnNode() { + setField(0); + } + + public int getField() { + return field; + } + + public SearchColumnNode setField(int field) { + distribution = new BucketDistribution(this.field = field, 16); + return this; + } + + public BucketDistribution getDistribution() { + return distribution; + } + + // Inherit doc from ExpressionNode. + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + // Inherit doc from ExpressionNode. + public Object evaluate(Context context) { + return distribution.getColumn(factory.getBucketId(context.getDocumentOperation().getId())); + } + + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return "searchcolumn." + field; + } + + public OrderingSpecification getOrdering(int order) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java b/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java index 77c462674db..f988729ebb8 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java @@ -1,58 +1,58 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.select.rule;
-
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSet;
-import com.yahoo.document.select.Context;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.document.select.Visitor;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class VariableNode implements ExpressionNode {
-
- private String value;
-
- public VariableNode(String value) {
- this.value = value;
- }
-
- public Object getValue() {
- return value;
- }
-
- public VariableNode setValue(String value) {
- this.value = value;
- return this;
- }
-
- @Override
- public BucketSet getBucketSet(BucketIdFactory factory) {
- return null;
- }
-
- @Override
- public Object evaluate(Context context) {
- Object o = context.getVariables().get(value);
- if (o == null) {
- throw new IllegalArgumentException("Variable " + value + " was not set in the variable list");
- }
- return o;
- }
-
- @Override
- public void accept(Visitor visitor) {
- visitor.visit(this);
- }
-
- @Override
- public String toString() {
- return "$" + value;
- }
-
- @Override
- public OrderingSpecification getOrdering(int order) {
- return null;
- }
-}
+package com.yahoo.document.select.rule; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSet; +import com.yahoo.document.select.Context; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.document.select.Visitor; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class VariableNode implements ExpressionNode { + + private String value; + + public VariableNode(String value) { + this.value = value; + } + + public Object getValue() { + return value; + } + + public VariableNode setValue(String value) { + this.value = value; + return this; + } + + @Override + public BucketSet getBucketSet(BucketIdFactory factory) { + return null; + } + + @Override + public Object evaluate(Context context) { + Object o = context.getVariables().get(value); + if (o == null) { + throw new IllegalArgumentException("Variable " + value + " was not set in the variable list"); + } + return o; + } + + @Override + public void accept(Visitor visitor) { + visitor.visit(this); + } + + @Override + public String toString() { + return "$" + value; + } + + @Override + public OrderingSpecification getOrdering(int order) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java index d6dd1ad35ac..e2c94c190ab 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java @@ -35,4 +35,4 @@ public class DocumentUpdateFlags { int mask = ~(~0 << 28); return combined & mask; } -}
\ No newline at end of file +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java index e97bd43d9bf..e385d0ce731 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java @@ -18,4 +18,4 @@ public interface FeedReader { */ public abstract void read(Operation operation) throws Exception; -}
\ No newline at end of file +} diff --git a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java index 9e0c4e353f2..8ba0bd815ea 100644 --- a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java +++ b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java @@ -1,44 +1,44 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document;
-
-import com.yahoo.document.datatypes.*;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for ticket 6394548
- */
-public class IncompatibleFieldTypesTest {
- private DataType arrayOfStrings;
- private StructDataType struct;
- private StructuredFieldValue root;
-
- @Before
- public void setUp() {
- arrayOfStrings = new ArrayDataType(DataType.STRING);
- struct = new StructDataType("fancypants");
- struct.addField(new Field("stringarray", arrayOfStrings, false));
- DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false);
- struct.addField(new Field("stringws", weightedSetOfStrings, false));
-
- root = struct.createFieldValue();
- root.setFieldValue("stringarray", arrayOfStrings.createFieldValue());
- root.setFieldValue("stringws", weightedSetOfStrings.createFieldValue());
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAddingIncompatibleFieldToArrayFails() {
- System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getClass().getName());
- System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getDataType().toString());
-
- ((Array)root.getFieldValue("stringarray")).add(new IntegerFieldValue(1234));
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testAddingIncompatibleFieldToWeightedSetFails() {
- System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getClass().getName());
- System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getDataType().toString());
-
- ((WeightedSet<FieldValue>)root.getFieldValue("stringws")).put(new IntegerFieldValue(1234), 100);
- }
-}
+package com.yahoo.document; + +import com.yahoo.document.datatypes.*; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests for ticket 6394548 + */ +public class IncompatibleFieldTypesTest { + private DataType arrayOfStrings; + private StructDataType struct; + private StructuredFieldValue root; + + @Before + public void setUp() { + arrayOfStrings = new ArrayDataType(DataType.STRING); + struct = new StructDataType("fancypants"); + struct.addField(new Field("stringarray", arrayOfStrings, false)); + DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false); + struct.addField(new Field("stringws", weightedSetOfStrings, false)); + + root = struct.createFieldValue(); + root.setFieldValue("stringarray", arrayOfStrings.createFieldValue()); + root.setFieldValue("stringws", weightedSetOfStrings.createFieldValue()); + } + + @Test(expected = IllegalArgumentException.class) + public void testAddingIncompatibleFieldToArrayFails() { + System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getClass().getName()); + System.out.println(root.getFieldValue("stringarray").getDataType().createFieldValue().getDataType().toString()); + + ((Array)root.getFieldValue("stringarray")).add(new IntegerFieldValue(1234)); + } + + @Test(expected = IllegalArgumentException.class) + public void testAddingIncompatibleFieldToWeightedSetFails() { + System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getClass().getName()); + System.out.println(root.getFieldValue("stringws").getDataType().createFieldValue().getDataType().toString()); + + ((WeightedSet<FieldValue>)root.getFieldValue("stringws")).put(new IntegerFieldValue(1234), 100); + } +} diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java index ee448bb79b0..6c007bad32a 100644 --- a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java @@ -1,123 +1,123 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.annotation;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.DocumentTypeManagerConfigurer;
-import com.yahoo.document.StructDataType;
-import com.yahoo.document.datatypes.*;
-import com.yahoo.document.serialization.*;
-import com.yahoo.io.GrowableByteBuffer;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Collections;
-
-import static org.junit.Assert.*;
-
-public class Bug6394548TestCase {
- @Test
- public void testSerializeAndDeserializeMultipleAdjacentStructAnnotations() {
- DocumentTypeManager manager = new DocumentTypeManager();
- DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg");
-
- AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry();
- AnnotationType featureSetType = registry.getType("morty.RICK_FEATURESET");
- assertNotNull(featureSetType);
-
- Document doc = new Document(manager.getDocumentType("article"), "doc:article:test");
- StringFieldValue sfv = new StringFieldValue("badger waltz");
-
- SpanList root = new SpanList();
- SpanNode node = new Span(0, sfv.getString().length());
- root.add(node);
-
- SpanTree tree = new SpanTree("rick_features", root);
- for (int i = 0; i < 2; ++i) {
- tree.annotate(createBigFeatureSetAnnotation(featureSetType));
- }
-
- sfv.setSpanTree(tree);
- doc.setFieldValue("title", sfv);
- System.out.println(doc.toXml());
- String annotationsBefore = dumpAllAnnotations(tree);
-
- GrowableByteBuffer buffer = new GrowableByteBuffer();
- DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
- serializer.write(doc);
-
- buffer.flip();
- DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer);
- Document doc2 = new Document(deserializer);
-
- System.out.println(doc2.toXml());
-
- StringFieldValue readString = (StringFieldValue)doc2.getFieldValue("title");
- SpanTree readTree = readString.getSpanTree("rick_features");
- assertNotNull(readTree);
- String annotationsAfter = dumpAllAnnotations(readTree);
-
- System.out.println("before:\n" + annotationsBefore);
- System.out.println("after:\n" + annotationsAfter);
-
- assertEquals(annotationsBefore, annotationsAfter);
- }
-
- private String dumpAllAnnotations(SpanTree tree) {
- ArrayList<String> tmp = new ArrayList<>();
- for (Annotation anno : tree) {
- Struct s = (Struct)anno.getFieldValue();
- tmp.add(s.toXml());
- }
- Collections.sort(tmp);
- StringBuilder annotations = new StringBuilder();
- for (String s : tmp) {
- annotations.append(s);
- }
- return annotations.toString();
- }
-
- private Annotation createBigFeatureSetAnnotation(AnnotationType featuresetAnno) {
- StructDataType featuresetType = (StructDataType)featuresetAnno.getDataType();
- Struct featureset = featuresetType.createFieldValue();
- System.out.println("featureset type: " + featureset.getDataType().toString());
-
- MapFieldValue<StringFieldValue, IntegerFieldValue> discreteValued
- = (MapFieldValue<StringFieldValue, IntegerFieldValue>)featuresetType.getField("discretevaluedfeatures").getDataType().createFieldValue();
- discreteValued.put(new StringFieldValue("foo"), new IntegerFieldValue(1234));
- discreteValued.put(new StringFieldValue("bar"), new IntegerFieldValue(567890123));
- featureset.setFieldValue("discretevaluedfeatures", discreteValued);
-
- MapFieldValue<StringFieldValue, DoubleFieldValue> realValued
- = (MapFieldValue<StringFieldValue, DoubleFieldValue>)featuresetType.getField("realvaluedfeatures").getDataType().createFieldValue();
- realValued.put(new StringFieldValue("foo"), new DoubleFieldValue(0.75));
- realValued.put(new StringFieldValue("bar"), new DoubleFieldValue(1.5));
- featureset.setFieldValue("realvaluedfeatures", realValued);
-
- Array<StringFieldValue> nested = (Array<StringFieldValue>)featureset.getField("foo10").getDataType().createFieldValue();
- nested.add(new StringFieldValue("baz"));
- nested.add(new StringFieldValue("blargh"));
- featureset.setFieldValue("foo10", nested);
-
- featureset.setFieldValue("foo1", new StringFieldValue("asdf"));
- featureset.setFieldValue("foo4", new StringFieldValue("qwerty"));
- featureset.setFieldValue("foo2", new IntegerFieldValue(555));
- featureset.setFieldValue("foo2", new IntegerFieldValue(8));
- featureset.setFieldValue("foo7", new IntegerFieldValue(1337));
- featureset.setFieldValue("foo8", new IntegerFieldValue(967867));
- featureset.setFieldValue("foo9", new DoubleFieldValue(123.45));
-
- Array<StringFieldValue> attributes = (Array<StringFieldValue>)featureset.getField("foo6").getDataType().createFieldValue();
- attributes.add(new StringFieldValue("adam"));
- attributes.add(new StringFieldValue("jamie"));
- attributes.add(new StringFieldValue("grant"));
- attributes.add(new StringFieldValue("tory"));
- attributes.add(new StringFieldValue("kari"));
- featureset.setFieldValue("variantattribute", attributes);
-
- Annotation anno = new Annotation(featuresetAnno);
- anno.setFieldValue(featureset);
-
- return anno;
- }
-}
+package com.yahoo.document.annotation; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.serialization.*; +import com.yahoo.io.GrowableByteBuffer; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; + +import static org.junit.Assert.*; + +public class Bug6394548TestCase { + @Test + public void testSerializeAndDeserializeMultipleAdjacentStructAnnotations() { + DocumentTypeManager manager = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(manager, "file:src/test/java/com/yahoo/document/annotation/documentmanager.6394548.cfg"); + + AnnotationTypeRegistry registry = manager.getAnnotationTypeRegistry(); + AnnotationType featureSetType = registry.getType("morty.RICK_FEATURESET"); + assertNotNull(featureSetType); + + Document doc = new Document(manager.getDocumentType("article"), "doc:article:test"); + StringFieldValue sfv = new StringFieldValue("badger waltz"); + + SpanList root = new SpanList(); + SpanNode node = new Span(0, sfv.getString().length()); + root.add(node); + + SpanTree tree = new SpanTree("rick_features", root); + for (int i = 0; i < 2; ++i) { + tree.annotate(createBigFeatureSetAnnotation(featureSetType)); + } + + sfv.setSpanTree(tree); + doc.setFieldValue("title", sfv); + System.out.println(doc.toXml()); + String annotationsBefore = dumpAllAnnotations(tree); + + GrowableByteBuffer buffer = new GrowableByteBuffer(); + DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + serializer.write(doc); + + buffer.flip(); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + Document doc2 = new Document(deserializer); + + System.out.println(doc2.toXml()); + + StringFieldValue readString = (StringFieldValue)doc2.getFieldValue("title"); + SpanTree readTree = readString.getSpanTree("rick_features"); + assertNotNull(readTree); + String annotationsAfter = dumpAllAnnotations(readTree); + + System.out.println("before:\n" + annotationsBefore); + System.out.println("after:\n" + annotationsAfter); + + assertEquals(annotationsBefore, annotationsAfter); + } + + private String dumpAllAnnotations(SpanTree tree) { + ArrayList<String> tmp = new ArrayList<>(); + for (Annotation anno : tree) { + Struct s = (Struct)anno.getFieldValue(); + tmp.add(s.toXml()); + } + Collections.sort(tmp); + StringBuilder annotations = new StringBuilder(); + for (String s : tmp) { + annotations.append(s); + } + return annotations.toString(); + } + + private Annotation createBigFeatureSetAnnotation(AnnotationType featuresetAnno) { + StructDataType featuresetType = (StructDataType)featuresetAnno.getDataType(); + Struct featureset = featuresetType.createFieldValue(); + System.out.println("featureset type: " + featureset.getDataType().toString()); + + MapFieldValue<StringFieldValue, IntegerFieldValue> discreteValued + = (MapFieldValue<StringFieldValue, IntegerFieldValue>)featuresetType.getField("discretevaluedfeatures").getDataType().createFieldValue(); + discreteValued.put(new StringFieldValue("foo"), new IntegerFieldValue(1234)); + discreteValued.put(new StringFieldValue("bar"), new IntegerFieldValue(567890123)); + featureset.setFieldValue("discretevaluedfeatures", discreteValued); + + MapFieldValue<StringFieldValue, DoubleFieldValue> realValued + = (MapFieldValue<StringFieldValue, DoubleFieldValue>)featuresetType.getField("realvaluedfeatures").getDataType().createFieldValue(); + realValued.put(new StringFieldValue("foo"), new DoubleFieldValue(0.75)); + realValued.put(new StringFieldValue("bar"), new DoubleFieldValue(1.5)); + featureset.setFieldValue("realvaluedfeatures", realValued); + + Array<StringFieldValue> nested = (Array<StringFieldValue>)featureset.getField("foo10").getDataType().createFieldValue(); + nested.add(new StringFieldValue("baz")); + nested.add(new StringFieldValue("blargh")); + featureset.setFieldValue("foo10", nested); + + featureset.setFieldValue("foo1", new StringFieldValue("asdf")); + featureset.setFieldValue("foo4", new StringFieldValue("qwerty")); + featureset.setFieldValue("foo2", new IntegerFieldValue(555)); + featureset.setFieldValue("foo2", new IntegerFieldValue(8)); + featureset.setFieldValue("foo7", new IntegerFieldValue(1337)); + featureset.setFieldValue("foo8", new IntegerFieldValue(967867)); + featureset.setFieldValue("foo9", new DoubleFieldValue(123.45)); + + Array<StringFieldValue> attributes = (Array<StringFieldValue>)featureset.getField("foo6").getDataType().createFieldValue(); + attributes.add(new StringFieldValue("adam")); + attributes.add(new StringFieldValue("jamie")); + attributes.add(new StringFieldValue("grant")); + attributes.add(new StringFieldValue("tory")); + attributes.add(new StringFieldValue("kari")); + featureset.setFieldValue("variantattribute", attributes); + + Annotation anno = new Annotation(featuresetAnno); + anno.setFieldValue(featureset); + + return anno; + } +} diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java index d34d0da8e82..68a97bc557c 100644 --- a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java @@ -1,66 +1,66 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.annotation;
-
-import com.yahoo.document.DataType;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.Field;
-import com.yahoo.document.StructDataType;
-import com.yahoo.document.datatypes.StringFieldValue;
-import com.yahoo.document.datatypes.Struct;
-import com.yahoo.document.serialization.*;
-import com.yahoo.io.GrowableByteBuffer;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class Bug6425939TestCase {
- private DocumentTypeManager man;
- private StructDataType person;
- private AnnotationType personA;
-
- @Before
- public void setUp() {
- man = new DocumentTypeManager();
-
- person = new StructDataType("personStruct");
- person.addField(new Field("foo", DataType.STRING));
- person.addField(new Field("bar", DataType.INT));
- man.register(person);
-
- personA = new AnnotationType("person", person);
- man.getAnnotationTypeRegistry().register(personA);
- }
-
- @Test
- public void canDeserializeAnnotationsOnZeroLengthStrings() {
- StringFieldValue emptyString = new StringFieldValue("");
- emptyString.setSpanTree(createSpanTree());
-
- GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
- DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
- Field strField = new Field("flarn", DataType.STRING);
- serializer.write(strField, emptyString);
- buffer.flip();
-
- // Should not throw exception if bug 6425939 is fixed:
- DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
- StringFieldValue deserializedString = new StringFieldValue();
- deserializer.read(strField, deserializedString);
-
- assertEquals("", deserializedString.getString());
- SpanTree readTree = deserializedString.getSpanTree("SpanTree1");
- assertNotNull(readTree);
- }
-
- private SpanTree createSpanTree() {
- SpanList root = new SpanList();
- SpanTree tree = new SpanTree("SpanTree1", root);
- SpanNode node = new Span(0, 0);
- Struct ps = new Struct(person);
- ps.setFieldValue("foo", "epic badger");
- ps.setFieldValue("bar", 54321);
- tree.annotate(node, new Annotation(personA, ps));
- root.add(node);
- return tree;
- }
-}
+package com.yahoo.document.annotation; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.serialization.*; +import com.yahoo.io.GrowableByteBuffer; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; + +public class Bug6425939TestCase { + private DocumentTypeManager man; + private StructDataType person; + private AnnotationType personA; + + @Before + public void setUp() { + man = new DocumentTypeManager(); + + person = new StructDataType("personStruct"); + person.addField(new Field("foo", DataType.STRING)); + person.addField(new Field("bar", DataType.INT)); + man.register(person); + + personA = new AnnotationType("person", person); + man.getAnnotationTypeRegistry().register(personA); + } + + @Test + public void canDeserializeAnnotationsOnZeroLengthStrings() { + StringFieldValue emptyString = new StringFieldValue(""); + emptyString.setSpanTree(createSpanTree()); + + GrowableByteBuffer buffer = new GrowableByteBuffer(1024); + DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + Field strField = new Field("flarn", DataType.STRING); + serializer.write(strField, emptyString); + buffer.flip(); + + // Should not throw exception if bug 6425939 is fixed: + DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + StringFieldValue deserializedString = new StringFieldValue(); + deserializer.read(strField, deserializedString); + + assertEquals("", deserializedString.getString()); + SpanTree readTree = deserializedString.getSpanTree("SpanTree1"); + assertNotNull(readTree); + } + + private SpanTree createSpanTree() { + SpanList root = new SpanList(); + SpanTree tree = new SpanTree("SpanTree1", root); + SpanNode node = new Span(0, 0); + Struct ps = new Struct(person); + ps.setFieldValue("foo", "epic badger"); + ps.setFieldValue("bar", 54321); + tree.annotate(node, new Annotation(personA, ps)); + root.add(node); + return tree; + } +} diff --git a/document/src/vespa/document/fieldvalue/variablemap.h b/document/src/vespa/document/fieldvalue/variablemap.h index 06679682e0c..ccebd3cdb52 100644 --- a/document/src/vespa/document/fieldvalue/variablemap.h +++ b/document/src/vespa/document/fieldvalue/variablemap.h @@ -44,4 +44,4 @@ public: vespalib::string toString() const; }; -}
\ No newline at end of file +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index de0ce196678..054c889dbfd 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -477,7 +477,12 @@ public class StoragePolicy extends ExternalSlobrokPolicy { @Override public void merge(RoutingContext context) { RoutingNodeIterator it = context.getChildIterator(); - Reply reply = it.removeReply(); + Reply reply = (it.hasReply()) ? it.removeReply() : context.getReply(); + if (reply == null) { + reply = new EmptyReply(); + reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, + "No reply in any children, nor in the routing context: " + context)); + } if (reply instanceof WrongDistributionReply) { distributorSelectionLogic.handleWrongDistribution((WrongDistributionReply) reply, context); diff --git a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp index 64e05fec793..b0ad75ffa3d 100644 --- a/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/systemstate/systemstate.cpp @@ -303,4 +303,4 @@ SystemState::SystemState(NodeState::UP root) : _lock(std::make_unique<vespalib::Lock>()) {} -SystemState::~SystemState() {}
\ No newline at end of file +SystemState::~SystemState() {} diff --git a/documentgen-test/etc/complex/parent.sd b/documentgen-test/etc/complex/parent.sd index 35bb650153a..09e4c5182b7 100644 --- a/documentgen-test/etc/complex/parent.sd +++ b/documentgen-test/etc/complex/parent.sd @@ -7,4 +7,4 @@ search parent { } } -}
\ No newline at end of file +} diff --git a/documentgen-test/pom.xml b/documentgen-test/pom.xml index 972cc8633f7..793aed44a74 100644 --- a/documentgen-test/pom.xml +++ b/documentgen-test/pom.xml @@ -75,7 +75,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp b/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp index 7f5f6bed1eb..8f5e3f4f71b 100644 --- a/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp @@ -39,4 +39,4 @@ Fast_FileOutputStream::Write(const void *sourceBuffer, size_t bufferSize) { bool Fast_FileOutputStream::Close() { return _theFile->Close(); -}
\ No newline at end of file +} diff --git a/fastlib/src/vespa/fastlib/testsuite/cpptest.el b/fastlib/src/vespa/fastlib/testsuite/cpptest.el index 6a1e1da09b0..d0a4f5c839a 100644 --- a/fastlib/src/vespa/fastlib/testsuite/cpptest.el +++ b/fastlib/src/vespa/fastlib/testsuite/cpptest.el @@ -2,7 +2,7 @@ ;; $Revision: 1.179 $ $Date: 2004-02-17 17:01:15 $ -;; Author: Nils Sandøy <nils.sandoy@fast.no> +;; Author: Nils Sandøy <nils.sandoy@fast.no> ;; Keywords: C++, tools ;; diff --git a/fastlib/src/vespa/fastlib/testsuite/testproject.el b/fastlib/src/vespa/fastlib/testsuite/testproject.el index 6cdfe9929ec..0a7ed652378 100644 --- a/fastlib/src/vespa/fastlib/testsuite/testproject.el +++ b/fastlib/src/vespa/fastlib/testsuite/testproject.el @@ -6,7 +6,7 @@ ;; the class(es) you want to test. ;; $Revision: 1.6 $ $Date: 2003-09-11 09:14:01 $ -;; Author: Nils Sandøy <nils.sandoy@fast.no> +;; Author: Nils Sandøy <nils.sandoy@fast.no> ;; Just a message to show that this file is beeing read. Look for this ;; in the *Messages* buffer. diff --git a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h index eb717ad489d..08b3e60a538 100644 --- a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h +++ b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h @@ -98,10 +98,12 @@ class WordFoldersTest : public Test } bool AccentRemovalTest() { + // Note last encoded characters encoded as octets to avoid interpreting following letters after xNN as part of the encoding of the character + // See http://en.cppreference.com/w/cpp/language/escape auto freefunction = [] (char * ptr) { free(ptr); }; - auto input = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþpþ!"), + auto input = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\x70\xFE\x21"), freefunction); - auto yelloutput = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿AAAAAEAAAECEEEEIIIIDNOOOOOE×OEUUUUEYTHssaaaaaeaaaeceeeeiiiidnoooooe÷oeuuuueythpth!"), + auto yelloutput = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\277AAAAAEAAAECEEEEIIIIDNOOOOOE\327OEUUUUEYTHssaaaaaeaaaeceeeeiiiidnoooooe\367oeuuuueythpth!"), freefunction); Fast_NormalizeWordFolder wordfolder; int len = wordfolder.FoldedSizeAsUTF8(input.get()); diff --git a/fastlib/src/vespa/fastlib/util/testproject.el b/fastlib/src/vespa/fastlib/util/testproject.el index f10d9ebc108..e0e2775d6ec 100644 --- a/fastlib/src/vespa/fastlib/util/testproject.el +++ b/fastlib/src/vespa/fastlib/util/testproject.el @@ -2,7 +2,7 @@ ;; Local configurations for the cpptest Emacs unit-test framework -;; Author: Nils Sandøy <nils.sandoy@fast.no> +;; Author: Nils Sandøy <nils.sandoy@fast.no> (message "Setting local test configuration") @@ -16,4 +16,4 @@ (setq cppt-test-parameters "") ;; Use -d to turn on debug mode -(setq cppt-test-dbflags "-d")
\ No newline at end of file +(setq cppt-test-dbflags "-d") diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java index 64cdd9da416..afbc879e40f 100644 --- a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java +++ b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java @@ -110,7 +110,12 @@ public class FileDistributionManager { public void reloadDeployFileDistributor() { try (LockGuard guard = new LockGuard(lock)) { - Runtime.getRuntime().exec("pkill -SIGUSR1 -x filedistributor"); + /* + * Try sending signal to vespa-filedistributor and + * filedistributor processes. Note 15 char limit for process + * name on Linux, see pkill manual page for details. + */ + Runtime.getRuntime().exec("pkill -SIGUSR1 ^(vespa-filedistr|filedistributor)$"); } catch (IOException e) { throw new RuntimeException("Failed to reinitialize the filedistributor", e); } diff --git a/fnet/RELEASEINFO b/fnet/RELEASEINFO deleted file mode 100644 index c4a53d9f1dc..00000000000 --- a/fnet/RELEASEINFO +++ /dev/null @@ -1,788 +0,0 @@ -======================================================================= - FNET CURRENT (2006-04-24) -....................................................................... - - -Misc ----- - -- (2007-01-12) Make enabling/disabling the direct write optimization - runtime rather than compile time. - - -Bugfixes --------- - -- (2007-01-09) Don't open new channels unless the channel id was - invented by our peer. Also, don't send a reply when an rpc request - error occurs. (avoid infinite error response loop) - -- (2006-09-21) Handle BAD_REQUEST/NULL method name case. When a bad - request was handled, the method name (NULL) would cause the method - lookup to crash. This should be resolved now. - -- On connection close: discard packets from the inner packet queue - even if the outer packet queue is empty. Not doing this could - trigger a resource leak if a dummy packet responsible for cleanup - (and channel closing) was left in the inner queue after the - connection was closed. - - -======================================================================= - FNET V_2_0_0_RELEASE -....................................................................... - - -New Dependencies ----------------- - -- vespalib (for test framework) - - -Removed Code/Features ---------------------- - -- Moved the core test framework into vespalib. FNET now depends on - vespalib for this. The FNET test framework is now implemented on top - of the test framework supplied by vespalib. Also, the vespalib test - script (testrun.sh) is used to control the running of FNET tests. - -- Removed FNET_LogHandler class. This removes the simple logging - framework located in FNET. It was deprecated several versions ago, - and all internal logging in FNET is now done using the new log - module. - -- Removed OO RPC Support (RPC with support for objects and - classes). This is done to simplify the overall design of RPC, which - also makes it simpler to implement compatible components in other - languages. RPC packet codes and error codes have been re-enumerated - to cover up the holes left by the removed support for OO RPC. The - protocol documentation has been modified to reflect the new RPC - design. NB: this makes RPC incompatible with older versions of FNET. - - -Spiral ------- - -Spiral is a simplified single-threaded RPC abstraction layer. It is -intended for truly single-threaded applications that does not want to -mess around with the complete FNET API. Spiral currently only supports -the RPC client aspect. - - -New Features ------------- - -- Added support for limiting the size of input/output buffers for - connections. This is done by invoking the - FNET_Transport::SetMaxInputBufferSize and - FNET_Transport::SetMaxOutputBufferSize methods. - -- Added FNET_Connection::GetQueueLen method that may be used to obtain the - number of packets currently located in the packet output queue. - -- Added support for enable/disable read and disable write events for - io components. This is used by the support to select on external - file descriptors. - -- Added support to shrink databuffers. This is used by the support to - limit input/output buffers. - -- Added support for selecting on external file descriptors by using - the FNET IO component framework. This includes using the - FNET_FDSelector class and the FNET_IFDSelectorHandler interface. - -- Better support for single-threaded compile. The test system has - been modified to work when compiled single-threaded. A new test - macro named TEST_THREADS may be used by tests that need threads to - run. This will result in the test begin skipped if threads are not - supported. - -- Added FRT_ISharedBlob interface with corresponding support. This may - be used to share the same memory regions between multiple - requests. Also, all shared blobs will be subref'ed by the request - as soon as the request packet is encoded to the connection output - buffer. - - -Bugfixes --------- - -- Fixed a minor bug in the xsync daemon; close other end of pipe after - fork, to avoid hang in mother process if child fails. - - -======================================================================= - FNET V_1_4_9_RELEASE -....................................................................... - - -Robustness ----------- - -- Make task scheduling more robust in cases where the event loop - latency is very high. (ref Bug #462022) - - -======================================================================= - FNET V_1_4_8_RELEASE -....................................................................... - - -======================================================================= - FNET V_1_4_7_RELEASE -....................................................................... - - -Misc ----- - -- Minor timing adjustments in the RPC invoke test to make it work - better. - - -======================================================================= - FNET V_1_4_6_RELEASE -....................................................................... - - -======================================================================= - FNET V_1_4_5_RELEASE -....................................................................... - - -======================================================================= - FNET V_1_4_4_RELEASE -....................................................................... - - -Misc ----- - -- avoid some warnings with gcc 4 (weffc++) - - -Bugfixes --------- - -- check if _adminChannel is closed while waiting for callbacks - (in the CloseAdminChannel method) - - -======================================================================= - FNET V_1_4_3_RELEASE -....................................................................... - - -Misc ----- - -- use vtag stuff in fastos to report fnet version - (to avoid empty release hassle) - -- Better support for make -j - - -======================================================================= - FNET V_1_4_2_RELEASE -....................................................................... - - -[NB: no changes at all; still identifies as 1.4.1] - - -======================================================================= - FNET V_1_4_1_RELEASE -....................................................................... - - -======================================================================= - FNET V_1_4_0_RELEASE -....................................................................... - - -Public testing toolkit ----------------------- - -The FNET testing toolkit has been made public and is now a part of the -fnet library. The macros have been renamed to all start with -'TEST_'. The local fnet tests have been modified and an example test -program has been added. - - -New logging API ---------------- - -All logging has been converted to use the new "log" module -and its API. All functions in class FNET_LogHandler are now -deprecated, in particular FNET_LogHandler::SetLogMask and -FNET_LogHandler::SetLogHandler no longer have any effect. -All users of fnet will need to link with the "log" library. - - -Misc ----- - -- thread_id test is void'ed if fnet is compiled without threads. - - -======================================================================= - FNET V_1_3_0_RELEASE -....................................................................... - - -FRT/RPC Lobotomy/Cleanup ------------------------- - -The support for local invocation (and location transparency) is -removed. The code is also somewhat simplified to reflect the now less -general nature of things. It hurts to see so much thought go down the -drain, but it greatly reduces maintenance complexity and simplifies -future development. Who needs sub microsecond invocations anyway... - -Changed code to avoid warning messages produced by gcc 3.3 and later -when compiling with the -Weffc++ option. - - -Removed Code ------------- - -- FNET_ITransportHook -- FNET_IPingable -- FNET_Pinger -- FNET_ServerInfo -- FNET_Transport::SetMinEventTimeOut -- FNET_Transport::SetPingInterval -- FNET_Transport::AddPingTarget -- FNET_Transport::RemovePingTarget -- FNET_Transport::HookNow - - -New Features ------------- - -- Added preliminary support for 2-way RPC. - -- Added support for invoking RPC methods without having an FRT_Target. - -- Added a Reset() method to the reflection manager that removes all - methods. - -- Added a method mismatch rpc hook that may be used to catch method - calls not matching any methods. - -- Added support for rpc session state (All rpc calls made on a single - connection is referred to as a session). This was done by adding a - context to each connection (used to store the session state), making - it possible to obtain the underlying connection given - the rpc request object and adding special rpc hooks that are called - when a connection is established, when a connection is lost and - right before a connection is deleted. - -- The transport object now comes with a built-in scheduler object. - -- The FNET_Scheduler constructor may be given the current time. This - simplifies running the scheduler in a simulated environment. - -- A Sync method has been added to the connection class. This method - may be used to block until all previously posted packets have been - encoded. - -- Added InTub method to the memorytub class. This method may be used - to check if the tub owns a given byte of memory. This feature is - primarily used for regression testing. - -- The FNET_Connector class has been given a GetPortNumber method to be - able to extract the port number of the underlying server socket. - -- Allow listening on port number 0. - -- Add FRT_Target::GetConnection() method. Only returns a pointer to - the connection owned by the target. No new references are allocated. - - -API changes ------------ - -- The FRT supervisor may be constructed either in stand-alone mode, or - based on an external transport object and an external thread - pool. The transport proxy methods may only be used when in - stand-alone mode. - -- The scheduler no longer implements the transport hook interface - (removed). The virtual Tick method has been replaced by the - non-virtual CheckTasks method. - -- The transport Main/Start methods no longer take a transport hook as - parameter (the interface has been removed). Similar functionality - may now be obtained by using the transport InitEventLoop and - EventLoopIteration methods. - -- The RPC target InvokeAsync method now returns bool, true means that - the method completed immediately. false means the invocation is in - progress. - -- The FRT_IRequestWait::RequestDone method now takes a second - parameter; a bool indicating whether this invocation is done - synchronously (directly from the invoke method). - - -Internal changes ----------------- - -- Added IOC CleanupHook method that is called right before an IOC is - deleted. This is used by the FNET_Connection class to support a - connection cleanup callback which in turn is used by the FRT_Supervisor to - activate the SessionFini hook needed for RPC session support. - -- Statistics updating is now done with a scheduled stats update - task. This reduces the complexity of the event loop. - -- The connection write method now avoids writing 0 bytes to the - network. Also, empty writes are silently ignored since using the - packet queue for control packets is perfectly legal. - -- Channel IDs are now chosen in a way that is future compatible with - 2-way channel creation. - - -Misc ----- - -- Store listen/connect spec in IOC and use it in IOC related log - messages. - -- Added rpc proxy example program that may be useful for debugging. - -- Added FNET_DummyPacket helper packet class that may be used to - implement hooks in the connection output packet queue. - -- The FNET_Packet::NoPCODE() and FNET_Connection::NoID() methods have - been replaced by the FNET_NOID macro. - -- Added FNET_ prefix to DISABLE_DIRECT_WRITE compile flag. - -- Added FNET_SANITY_CHECKS compile flag. This enables some extra - sanity checks in the event loop. - -- Added generic cross-host testing support as part of testing - framework (xtest make target removed). This includes the xsync - daemon used to sync test states between hosts. The environment - variables PORTBASE and PEER are used to control testing behavior. - -- In regression test framework: enable output from passed test cases - to be collapsed to reduce logging. - -- Added support for mutex instrumentation. This requires a version of - FastOS supporting instrumented mutexes. - -- Added FRT_Supervisor::GetListenPort() method. - - -Bugfixes --------- - -- RPC: Handle request return (after request detach) while still - inside sync invocation method correctly (also added to regress - test). - -- RPC: ensure that the cleanup handler is run also when the request is - recycled by the user without making the trip back to the request - pool. - - -======================================================================= - FNET V_1_2_4_RELEASE -....................................................................... - - -Bugfixes --------- - -- Wait for direct write to complete before discarding packets when the - connection is going down. - - -======================================================================= - FNET V_1_2_3_RELEASE -....................................................................... - - -New Features ------------- - -- Added support for sending multiple packets at the same time. This - may reduce TCP latency when used with Nagle's algorithm. - - -======================================================================= - FNET V_1_2_2_RELEASE -....................................................................... - - -Bugfixes --------- - -- Fix Makefile to work with parallel compilation (gcc -j option). - - -======================================================================= - FNET V_1_2_1_RELEASE -....................................................................... - - -Misc ----- - -- Max events per event loop iteration: 256 -> 4096 - - -Bugfixes --------- - -- Avoid overlap between chunk header and chunk data in FRT memorytub. - -- Remove race around single req wait where condition could be - destructed while being locked. - -- Don't set connection state to closed if connect fails - synchronously. This caused a channel lost event to be sent on the - admin channel that caused a deadlock in the serverinfo class. It is - also an API improvement, since connect now fails either - synchronously or asynchronously (not both). - - -======================================================================= - FNET V_1_2_0_RELEASE -....................................................................... - - -New Features ------------- - -- Added support to construct an FNET_DataBuffer based on a - preallocated buffer. - -- Added support for scheduling a task to be run as soon as possible. - -- Added support for killing a task so that it may not be scheduled in - the future. - -- Added databuffer methods to write data without checking for free - space. These methods have a 'Fast' suffix. - -- Added AssertValid method that may be called anytime to assert that - the databuffer is in a legal state. - -- Added endian detection. Use the FNET_Info::GetEndian method to - obtain host endian. - -- Added methods to read/peek information stored in reverse internet - order (little endian) in a databuffer. - -- VPATH compilation. - - -Internal Changes ----------------- - -- Removed asserts used to check for valid data in databuffer read - methods. - -- When posting a packet, try to write it to the network directly (if - there is no other pending write operations). Post - an enable write event to the transport thread if there is pending - work after a direct write has been attempted. This behavior may be - disabled by using the --disable-direct-write configure option. - - -Misc ----- - -- Added static instance of FNET_CMD_NOCOMMAND control packet for - completeness. - -- Moved fastserver stuff to the fastserver4 cvs module. - -- Simple regress test framework. Run tests with 'make test' from - fnet/src. Use 'make xtest' to run cross-host testing. This requires - setting the PORT and PEER environment variables to sane - value. Example: to run a cross-host test between host A and host B - using port 8000 (on both hosts); run 'make PORT=8000 PEER=B xtest' - on host A and 'make PORT=8000 PEER=A xtest' on host B. - - -Bugfixes --------- - -- Set connection state correctly if socket connect completes - synchronously. - - -FRT ([F]NET [R]emote [T]ools) [middle-ware library] ---------------------------------------------------- - -The FRT library is a layer on top of FNET supporting a proprietary RPC -protocol. In addition to the network protocol, the FRT library also -contains API support for location transparent dynamic method -invocation (examples and regress tests also included). - - -======================================================================= - FNET V_1_1_2_RELEASE -....................................................................... - - -Bugfixes --------- - -- Handle EOL returned from socket read. This lets FNET handle peers - that close connections nicely. Previously this would result in a - very tight event loop reading EOL from the socket in each iteration. - -- Avoid time measurement errors in scheduler by using absolute values - for bucket timeouts rather than accumulated delta values. If the - scheduler was invoked very often, it would appear that time stood - still. - - -======================================================================= - FNET V_1_1_1_RELEASE -....................................................................... - - -Compiler nitpicks ------------------ - -- Renamed TICK to SLOT_TICK in scheduler class to avoid conflict with - macro on Solaris. -- Removed comma at end of enums in fs4 example packet library. -- Use public inheritance in fsearch_maxthreads fs4 example - application. -- Conceal infinite loop in fsearch_somethreads fs4 example - application. -- Cast error-code to signed value before applying negation in - fs4 searchtest example application. -- Cast enum to uint32_t to avoid constructor confusion in proxy - example application. -- Explicitly cast stat values to float. - - -Misc ----- - -- Removed memory leak in scheduling test to avoid HUGE memory leak - report on application exit. - - -======================================================================= - FNET V_1_1_0_RELEASE -....................................................................... - - -API Changes ------------ - -Added method bool FNET_IServerAdapter::InitAdminChannel(FNET_Channel -*) That must be implemented by server applications using FNET. The -method is called when a new incoming connection has been established -in order to let the application keep track of its current -clients. This channel works just like normal channels. The only -difference is that it is opened before any packets are received, -thereby allowing the application to implement connection-oriented -stuff like proxies. Implementing this method to always return false -will preserve old behavior. NOTE: closing the connection directly (by -invoking Close()) is allowed from the InitAdminChannel method (This is -an exception from the general rule that the application should close -all connections with async events. The reason for this exception is to -allow better resource management). - -Added an FNET_Context parameter to the FNET_Connection constructor and -to the FNET_Transport::Connect method. This was done to enable the -application to set the admin channel context in the client end-point -of a connection. Also, VOIDP=NULL is used as default context, instead -of the previously used INT=0xffffffff (FNET_Connection::NoID()). - -Added a parameter of type FNET_ITransportHook to the -FNET_Transport::Start method in order to enable sharing the transport -thread without the need to create a wrapper object (The parameter is -optional with NULL as default value). - -Made the FNET_ITransportHook parameter to the FNET_Transport::Main -method optional (default value is NULL). - -Server info objects may now be marked as 'bad' -[MarkBad()/ClearBad()/IsBad()]. If a server (info) is marked as bad, -FNET will try to keep it down, rather than up. Note that node badness -may be set by the application code in any thread and is not protected -by any locks. The most sensible use of this flag would be to use it as -an async signal, raising it if a server goes bad, and never lowering -it again (ClearBad() is implemented for completeness, but needs to be -used with care). - -Added new method FNET_Channel::CloseAndFree() that may be used to -close and free a channel in a single operation. This is equivalent -with first invoking Close(), then Free(), but is more efficient. - -The packet streamer interface has been modified in order to support -context-dependent un-streaming of packets and variable length packet -headers. This has resulted in the following method changes: - ------ -bool -FNET_IServerAdapter::InitChannel(FNET_Channel *channel, - FNET_Packet *packet) ------ -|| -\/ ------ -bool -FNET_IServerAdapter::InitChannel(FNET_Channel *channel, - uint32_t pcode) ------ - -This reflects the fact that in order to do context-dependent packet -un-streaming, we need to resolve the context of the channel that -should receive the packet before un-streaming the packet. This also -means that the packet itself is not available when opening server-side -channels. - ------ -FNET_Packet * -FNET_IPacketFactory::CreatePacket(uint32_t pcode) ------ -|| -\/ ------ -FNET_Packet * -FNET_IPacketFactory::CreatePacket(uint32_t pcode, FNET_Context context) ------ - -The context is propagated to the packet factory in order to eliminate -the need to change the FNET_Packet::Decode(...) method. - ------ -uint32_t -FNET_IPacketStreamer::GetHeaderLen() - -uint32_t -FNET_IPacketStreamer::GetPacketLen(FNET_DataBuffer *src) ------ -|| -\/ ------ -bool -FNET_IPacketStreamer::GetPacketInfo(FNET_DataBuffer *src, - uint32_t *plen, - uint32_t *pcode, - uint32_t *chid, - bool *broken) ------ - -The packet info is extracted before the packet is un-streamed in order -to locate the channel that should receive the packet. This change -gives us support for variable length packet headers and a way to -signal that the connection is broken. - ------ -FNET_Packet * -FNET_IPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t *chid) ------ -|| -\/ ------ -FNET_Packet * -FNET_IPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, - uint32_t pcode, FNET_Context context) ------ - -The packet info is injected as parameters to avoid having to extract -it from the packet a second time. Note that the chid has been replaced -by the context of the channel that will receive the packet. This -completes the support for context-dependent packet un-streaming. - - -Internal Changes ----------------- - -Allow IOC_DELETE event before IOC_ADD event. This enables the -application to post an event indicating that the connection itself -should be closed when invoked to open the admin channel on that -connection (only applies to server connections). - -Optimized FNET_DataBuffer by using pointers to define dead/data/free -sections of the buffer. - -Each connection now has its own FNET_ChannelPool object instead of -sharing an FNET_ObjectPool. This reduces locking. - -The methods on FNET_Channel has been un-inlined in order to -instantiate FNET_Connection objects with an inlined FNET_ChannelPool -object which in turn has inline FNET_Channel objects. - -Make sure that the packets posted on a connection are freed in the -same order as they were posted, even if the connection goes down. This -is done to enable the application to use packet free order for syncing -purposes. It also enables the usage of packets that perform code in -the free method that requires that all previously posted packets have -already been freed (Like memory pool cleanup code). - -Changed some log messages to FNET_LOG_DEBUG. - - -New Control Packets -------------------- - -- FNET_CMD_TIMEOUT : This control packet command may be used to signal - a timeout. Note that FNET does not use timeout - packets internally. They are only included to - ease the implementation of timeout signaling in - applications using FNET. - -- FNET_CMD_BAD_PACKET : This control packet signals the reception of - a packet that could not be un-streamed. It - will be delivered in place of the packet that - was invalid. - - -New Classes ------------ - -- FNET_Task (schedulable tasks; timeouts etc.) -- FNET_Scheduler (class for scheduling tasks) -- FNET_ChannelPool (per-connection channel pooling) - - -Removed Classes ---------------- - -- FNET_ObjectPool - - -New Examples ------------- - -- proxy : A simple proxy implementation showing how FNET may be used - to handle raw data as packets and also how to focus on - connections rather than channels. - -- timeout : Simple application showing how to use the - task/scheduler classes to implement timeout support. - - -Bugfixes --------- - -- use maxreconnectdelay if the connect failed synchronously. - -- don't try to connect to target host if basic socket init fails. - - -======================================================================= - FNET V_1_0_0_RELEASE -....................................................................... - - -Initial release. - -NOTE: 1.0.x is no longer supported. diff --git a/fnet/src/vespa/fnet/frt/target.cpp b/fnet/src/vespa/fnet/frt/target.cpp index 11e8515719e..15821856525 100644 --- a/fnet/src/vespa/fnet/frt/target.cpp +++ b/fnet/src/vespa/fnet/frt/target.cpp @@ -27,4 +27,4 @@ FRT_Target::InvokeVoid(FRT_RPCRequest *req) { void FRT_Target::InvokeSync(FRT_RPCRequest *req, double timeout) { FRT_Supervisor::InvokeSync(_scheduler, _conn, req, timeout); -}
\ No newline at end of file +} diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp index dd7a3246090..404024daf7f 100644 --- a/fnet/src/vespa/fnet/scheduler.cpp +++ b/fnet/src/vespa/fnet/scheduler.cpp @@ -277,4 +277,4 @@ FNET_Scheduler::PerformTasks(uint32_t slot, uint32_t iter) { bool FNET_Scheduler::IsActive(FNET_Task *task) { return task->_task_next != nullptr; -}
\ No newline at end of file +} diff --git a/fsa/TODO b/fsa/TODO deleted file mode 100644 index 7244fc3f002..00000000000 --- a/fsa/TODO +++ /dev/null @@ -1,7 +0,0 @@ -* Improve performance of traversing fsa (iterator) by limiting the number of - possible transitions which need to be checked. - -* Extend makefsa -v (verbose) option to include statistics (memory usage etc.) - -* Make memory consumption and allocation strategy in Automaton more efficient. - diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java index f648a0e38e4..9596bf60cae 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java @@ -1,55 +1,55 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.indexinglanguage.expressions;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Stack;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class MathResolver {
-
- private final List<Item> items = new LinkedList<>();
-
- public void push(ArithmeticExpression.Operator op, Expression exp) {
- op.getClass(); // throws NullPointerException
- if (items.isEmpty() && op != ArithmeticExpression.Operator.ADD) {
- throw new IllegalArgumentException("First item in an arithmetic operation must be an addition.");
- }
- items.add(new Item(op, exp));
- }
-
- public Expression resolve() {
- Stack<Item> stack = new Stack<>();
- stack.push(items.remove(0));
- while (!items.isEmpty()) {
- Item item = items.remove(0);
- while (stack.size() > 1 && stack.peek().op.precedes(item.op)) {
- pop(stack);
- }
- stack.push(item);
- }
- while (stack.size() > 1) {
- pop(stack);
- }
- return stack.remove(0).exp;
- }
-
- private void pop(Stack<Item> stack) {
- Item rhs = stack.pop();
- Item lhs = stack.peek();
- lhs.exp = new ArithmeticExpression(lhs.exp, rhs.op, rhs.exp);
- }
-
- private static class Item {
-
- final ArithmeticExpression.Operator op;
- Expression exp;
-
- Item(ArithmeticExpression.Operator op, Expression exp) {
- this.op = op;
- this.exp = exp;
- }
- }
-}
\ No newline at end of file +package com.yahoo.vespa.indexinglanguage.expressions; + +import java.util.LinkedList; +import java.util.List; +import java.util.Stack; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class MathResolver { + + private final List<Item> items = new LinkedList<>(); + + public void push(ArithmeticExpression.Operator op, Expression exp) { + op.getClass(); // throws NullPointerException + if (items.isEmpty() && op != ArithmeticExpression.Operator.ADD) { + throw new IllegalArgumentException("First item in an arithmetic operation must be an addition."); + } + items.add(new Item(op, exp)); + } + + public Expression resolve() { + Stack<Item> stack = new Stack<>(); + stack.push(items.remove(0)); + while (!items.isEmpty()) { + Item item = items.remove(0); + while (stack.size() > 1 && stack.peek().op.precedes(item.op)) { + pop(stack); + } + stack.push(item); + } + while (stack.size() > 1) { + pop(stack); + } + return stack.remove(0).exp; + } + + private void pop(Stack<Item> stack) { + Item rhs = stack.pop(); + Item lhs = stack.peek(); + lhs.exp = new ArithmeticExpression(lhs.exp, rhs.op, rhs.exp); + } + + private static class Item { + + final ArithmeticExpression.Operator op; + Expression exp; + + Item(ArithmeticExpression.Operator op, Expression exp) { + this.op = op; + this.exp = exp; + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java index 427c777db5a..703cf30f6e0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java @@ -18,4 +18,4 @@ public class PassthroughExpression extends OutputExpression { public boolean equals(Object obj) { return super.equals(obj) && obj instanceof PassthroughExpression; } -}
\ No newline at end of file +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java index 09364c796f0..48e78392498 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java @@ -1,106 +1,106 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.indexinglanguage.linguistics;
-
-import com.yahoo.language.Language;
-import com.yahoo.language.process.StemMode;
-import com.yahoo.vespa.configdefinition.IlscriptsConfig;
-
-/**
- * @author Simon Thoresen
- */
-public class AnnotatorConfig implements Cloneable {
-
- private Language language;
- private StemMode stemMode;
- private boolean removeAccents;
- private int maxTermOccurences;
-
- public static final int DEFAULT_MAX_TERM_OCCURRENCES;
-
- static {
- IlscriptsConfig defaults = new IlscriptsConfig(new IlscriptsConfig.Builder());
- DEFAULT_MAX_TERM_OCCURRENCES = defaults.maxtermoccurrences();
- }
-
- public AnnotatorConfig() {
- language = Language.ENGLISH;
- stemMode = StemMode.NONE;
- removeAccents = false;
- maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES;
- }
-
- public AnnotatorConfig(AnnotatorConfig rhs) {
- language = rhs.language;
- stemMode = rhs.stemMode;
- removeAccents = rhs.removeAccents;
- maxTermOccurences = rhs.maxTermOccurences;
- }
-
- public Language getLanguage() {
- return language;
- }
-
- public AnnotatorConfig setLanguage(Language language) {
- this.language = language;
- return this;
- }
-
- public StemMode getStemMode() {
- return stemMode;
- }
-
- public AnnotatorConfig setStemMode(StemMode stemMode) {
- this.stemMode = stemMode;
- return this;
- }
-
- public AnnotatorConfig setStemMode(String name) {
- this.stemMode = StemMode.valueOf(name);
- return this;
- }
-
- public boolean getRemoveAccents() {
- return removeAccents;
- }
-
- public AnnotatorConfig setRemoveAccents(boolean removeAccents) {
- this.removeAccents = removeAccents;
- return this;
- }
-
- public int getMaxTermOccurrences() {
- return maxTermOccurences;
- }
-
- public AnnotatorConfig setMaxTermOccurrences(int maxTermCount) {
- this.maxTermOccurences = maxTermCount;
- return this;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof AnnotatorConfig)) {
- return false;
- }
- AnnotatorConfig rhs = (AnnotatorConfig)obj;
- if (!language.equals(rhs.language)) {
- return false;
- }
- if (!stemMode.equals(rhs.stemMode)) {
- return false;
- }
- if (removeAccents != rhs.removeAccents) {
- return false;
- }
- if (maxTermOccurences != rhs.maxTermOccurences) {
- return false;
- }
- return true;
- }
-
- @Override
- public int hashCode() {
- return getClass().hashCode() + language.hashCode() + stemMode.hashCode() +
- Boolean.valueOf(removeAccents).hashCode() + maxTermOccurences;
- }
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.linguistics; + +import com.yahoo.language.Language; +import com.yahoo.language.process.StemMode; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; + +/** + * @author Simon Thoresen + */ +public class AnnotatorConfig implements Cloneable { + + private Language language; + private StemMode stemMode; + private boolean removeAccents; + private int maxTermOccurences; + + public static final int DEFAULT_MAX_TERM_OCCURRENCES; + + static { + IlscriptsConfig defaults = new IlscriptsConfig(new IlscriptsConfig.Builder()); + DEFAULT_MAX_TERM_OCCURRENCES = defaults.maxtermoccurrences(); + } + + public AnnotatorConfig() { + language = Language.ENGLISH; + stemMode = StemMode.NONE; + removeAccents = false; + maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES; + } + + public AnnotatorConfig(AnnotatorConfig rhs) { + language = rhs.language; + stemMode = rhs.stemMode; + removeAccents = rhs.removeAccents; + maxTermOccurences = rhs.maxTermOccurences; + } + + public Language getLanguage() { + return language; + } + + public AnnotatorConfig setLanguage(Language language) { + this.language = language; + return this; + } + + public StemMode getStemMode() { + return stemMode; + } + + public AnnotatorConfig setStemMode(StemMode stemMode) { + this.stemMode = stemMode; + return this; + } + + public AnnotatorConfig setStemMode(String name) { + this.stemMode = StemMode.valueOf(name); + return this; + } + + public boolean getRemoveAccents() { + return removeAccents; + } + + public AnnotatorConfig setRemoveAccents(boolean removeAccents) { + this.removeAccents = removeAccents; + return this; + } + + public int getMaxTermOccurrences() { + return maxTermOccurences; + } + + public AnnotatorConfig setMaxTermOccurrences(int maxTermCount) { + this.maxTermOccurences = maxTermCount; + return this; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof AnnotatorConfig)) { + return false; + } + AnnotatorConfig rhs = (AnnotatorConfig)obj; + if (!language.equals(rhs.language)) { + return false; + } + if (!stemMode.equals(rhs.stemMode)) { + return false; + } + if (removeAccents != rhs.removeAccents) { + return false; + } + if (maxTermOccurences != rhs.maxTermOccurences) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + language.hashCode() + stemMode.hashCode() + + Boolean.valueOf(removeAccents).hashCode() + maxTermOccurences; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java index de9d26547da..5e9ffd677cf 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java @@ -1,14 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.indexinglanguage.parser;
-
-import com.yahoo.javacc.FastCharStream;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public final class IndexingInput extends FastCharStream implements CharStream {
-
- public IndexingInput(String input) {
- super(input);
- }
-}
+package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.javacc.FastCharStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public final class IndexingInput extends FastCharStream implements CharStream { + + public IndexingInput(String input) { + super(input); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java index 097e0f21bc1..95eab5f9ef8 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java @@ -38,4 +38,4 @@ public class AttributeExpressionTestCase { public void requireThatExpressionCanBeExecuted() { assertExecute(new AttributeExpression("foo")); } -}
\ No newline at end of file +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index 805bdc96904..5882e2c19c6 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -1,250 +1,250 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.indexinglanguage.linguistics;
-
-import com.yahoo.document.annotation.Annotation;
-import com.yahoo.document.annotation.AnnotationTypes;
-import com.yahoo.document.annotation.SpanTree;
-import com.yahoo.document.annotation.SpanTrees;
-import com.yahoo.document.datatypes.StringFieldValue;
-import com.yahoo.language.Language;
-import com.yahoo.language.Linguistics;
-import com.yahoo.language.process.StemMode;
-import com.yahoo.language.process.Token;
-import com.yahoo.language.process.TokenType;
-import com.yahoo.language.process.Tokenizer;
-import com.yahoo.language.simple.SimpleToken;
-
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.util.*;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class LinguisticsAnnotatorTestCase {
-
- private static final AnnotatorConfig CONFIG = new AnnotatorConfig();
-
- // --------------------------------------------------------------------------------
- //
- // Tests
- //
- // --------------------------------------------------------------------------------
-
- @Test
- public void requireThatAnnotateFailsWithZeroTokens() {
- assertAnnotations(null, "foo");
- }
-
- @Test
- public void requireThatAnnotateFailsWithoutIndexableTokenString() {
- for (TokenType type : TokenType.values()) {
- if (type.isIndexable()) {
- continue;
- }
- assertAnnotations(null, "foo", newToken("foo", "bar", type));
- }
- }
-
- @Test
- public void requireThatIndexableTokenStringsAreAnnotated() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
- for (TokenType type : TokenType.values()) {
- if (!type.isIndexable()) {
- continue;
- }
- assertAnnotations(expected, "foo", newToken("foo", "bar", type));
- }
- }
-
- @Test
- public void requireThatSpecialTokenStringsAreAnnotatedRegardlessOfType() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
- for (TokenType type : TokenType.values()) {
- assertAnnotations(expected, "foo", newToken("foo", "bar", type, true));
- }
- }
-
- @Test
- public void requireThatTermAnnotationsAreEmptyIfOrigIsLowerCase() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM));
- for (boolean specialToken : Arrays.asList(true, false)) {
- for (TokenType type : TokenType.values()) {
- if (!specialToken && !type.isIndexable()) {
- continue;
- }
- assertAnnotations(expected, "foo", newToken("foo", "foo", type, specialToken));
- }
- }
- }
-
- @Test
- public void requireThatTermAnnotationsAreLowerCased() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
- for (boolean specialToken : Arrays.asList(true, false)) {
- for (TokenType type : TokenType.values()) {
- if (!specialToken && !type.isIndexable()) {
- continue;
- }
- assertAnnotations(expected, "foo", newToken("foo", "BAR", type, specialToken));
- }
- }
- }
-
- @Test
- public void requireThatCompositeTokensAreFlattened() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("foo")));
- expected.spanList().span(3, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
- expected.spanList().span(6, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));
-
- SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC)
- .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0))
- .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3)
- .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3))
- .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6)));
- assertAnnotations(expected, "foobarbaz", token);
- }
-
- @Test
- public void requireThatCompositeSpecialTokensAreNotFlattened() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 9).annotate(new Annotation(AnnotationTypes.TERM,
- new StringFieldValue("foobarbaz")));
-
- SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true)
- .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0))
- .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3)
- .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3))
- .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6)));
- assertAnnotations(expected, "foobarbaz", token);
- }
-
- @Test
- public void requireThatErrorTokensAreSkipped() {
- assertAnnotations(null, "foo", new SimpleToken("foo").setType(TokenType.ALPHABETIC)
- .setOffset(-1));
- }
-
- @Test
- public void requireThatTermReplacementsAreApplied() {
- SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
- for (boolean specialToken : Arrays.asList(true, false)) {
- for (TokenType type : TokenType.values()) {
- if (!specialToken && !type.isIndexable()) {
- continue;
- }
- assertAnnotations(expected, "foo",
- newLinguistics(Arrays.asList(newToken("foo", "foo", type, specialToken)),
- Collections.singletonMap("foo", "bar")));
- }
- }
- }
-
- @Test
- public void requireThatExistingAnnotationsAreKept() {
- SpanTree spanTree = new SpanTree(SpanTrees.LINGUISTICS);
- spanTree.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));
-
- StringFieldValue val = new StringFieldValue("foo");
- val.setSpanTree(spanTree);
-
- Linguistics linguistics = newLinguistics(Arrays.asList(newToken("foo", "bar", TokenType.ALPHABETIC, false)),
- Collections.<String, String>emptyMap());
- new LinguisticsAnnotator(linguistics, CONFIG).annotate(val);
-
- assertTrue(new LinguisticsAnnotator(linguistics, CONFIG).annotate(val));
- assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS));
- }
-
- @Test
- public void requireThatMaxTermOccurencesIsHonored() {
- final String inputTerm = "foo";
- final String stemmedInputTerm = "bar"; // completely different from
- // inputTerm for safer test
- final String paddedInputTerm = inputTerm + " ";
- final SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
- final int inputTermOccurence = AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES * 2;
- for (int i = 0; i < AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES; ++i) {
- expected.spanList().span(i * paddedInputTerm.length(), inputTerm.length())
- .annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue(stemmedInputTerm)));
- }
- for (TokenType type : TokenType.values()) {
- if (!type.isIndexable()) {
- continue;
- }
- StringBuilder input = new StringBuilder();
- Token[] tokens = new Token[inputTermOccurence];
- for (int i = 0; i < inputTermOccurence; ++i) {
- SimpleToken t = newToken(inputTerm, stemmedInputTerm, type);
- t.setOffset(i * paddedInputTerm.length());
- tokens[i] = t;
- input.append(paddedInputTerm);
- }
- assertAnnotations(expected, input.toString(), tokens);
- }
- }
-
- // --------------------------------------------------------------------------------
- //
- // Utilities
- //
- // --------------------------------------------------------------------------------
-
- private static SimpleToken newToken(String orig, String stem, TokenType type) {
- return newToken(orig, stem, type, false);
- }
-
- private static SimpleToken newToken(String orig, String stem, TokenType type, boolean specialToken) {
- return new SimpleToken(orig).setTokenString(stem)
- .setType(type)
- .setSpecialToken(specialToken);
- }
-
- private static void assertAnnotations(SpanTree expected, String value, Token... tokens) {
- assertAnnotations(expected, value, newLinguistics(Arrays.asList(tokens), Collections.<String, String>emptyMap()));
- }
-
- private static void assertAnnotations(SpanTree expected, String str, Linguistics linguistics) {
- StringFieldValue val = new StringFieldValue(str);
- assertEquals(expected != null, new LinguisticsAnnotator(linguistics, CONFIG).annotate(val));
- assertEquals(expected, val.getSpanTree(SpanTrees.LINGUISTICS));
- }
-
- private static Linguistics newLinguistics(List<? extends Token> tokens, Map<String, String> replacementTerms) {
- Linguistics linguistics = Mockito.mock(Linguistics.class);
- Mockito.when(linguistics.getTokenizer()).thenReturn(new MyTokenizer(tokens, replacementTerms));
- return linguistics;
- }
-
- private static class MyTokenizer implements Tokenizer {
-
- final List<Token> tokens;
- final Map<String, String> replacementTerms;
-
- public MyTokenizer(List<? extends Token> tokens, Map<String, String> replacementTerms) {
- this.tokens = new ArrayList<>(tokens);
- this.replacementTerms = replacementTerms;
- }
-
- @Override
- public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
- return tokens;
- }
-
- @Override
- public String getReplacementTerm(String term) {
- String replacement = replacementTerms.get(term);
- return replacement != null ? replacement : term;
- }
- }
-}
+package com.yahoo.vespa.indexinglanguage.linguistics; + +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationTypes; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.annotation.SpanTrees; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; +import com.yahoo.language.process.TokenType; +import com.yahoo.language.process.Tokenizer; +import com.yahoo.language.simple.SimpleToken; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LinguisticsAnnotatorTestCase { + + private static final AnnotatorConfig CONFIG = new AnnotatorConfig(); + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatAnnotateFailsWithZeroTokens() { + assertAnnotations(null, "foo"); + } + + @Test + public void requireThatAnnotateFailsWithoutIndexableTokenString() { + for (TokenType type : TokenType.values()) { + if (type.isIndexable()) { + continue; + } + assertAnnotations(null, "foo", newToken("foo", "bar", type)); + } + } + + @Test + public void requireThatIndexableTokenStringsAreAnnotated() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + for (TokenType type : TokenType.values()) { + if (!type.isIndexable()) { + continue; + } + assertAnnotations(expected, "foo", newToken("foo", "bar", type)); + } + } + + @Test + public void requireThatSpecialTokenStringsAreAnnotatedRegardlessOfType() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + for (TokenType type : TokenType.values()) { + assertAnnotations(expected, "foo", newToken("foo", "bar", type, true)); + } + } + + @Test + public void requireThatTermAnnotationsAreEmptyIfOrigIsLowerCase() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM)); + for (boolean specialToken : Arrays.asList(true, false)) { + for (TokenType type : TokenType.values()) { + if (!specialToken && !type.isIndexable()) { + continue; + } + assertAnnotations(expected, "foo", newToken("foo", "foo", type, specialToken)); + } + } + } + + @Test + public void requireThatTermAnnotationsAreLowerCased() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + for (boolean specialToken : Arrays.asList(true, false)) { + for (TokenType type : TokenType.values()) { + if (!specialToken && !type.isIndexable()) { + continue; + } + assertAnnotations(expected, "foo", newToken("foo", "BAR", type, specialToken)); + } + } + } + + @Test + public void requireThatCompositeTokensAreFlattened() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("foo"))); + expected.spanList().span(3, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + expected.spanList().span(6, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz"))); + + SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC) + .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) + .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) + .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) + .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); + assertAnnotations(expected, "foobarbaz", token); + } + + @Test + public void requireThatCompositeSpecialTokensAreNotFlattened() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 9).annotate(new Annotation(AnnotationTypes.TERM, + new StringFieldValue("foobarbaz"))); + + SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true) + .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) + .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) + .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) + .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); + assertAnnotations(expected, "foobarbaz", token); + } + + @Test + public void requireThatErrorTokensAreSkipped() { + assertAnnotations(null, "foo", new SimpleToken("foo").setType(TokenType.ALPHABETIC) + .setOffset(-1)); + } + + @Test + public void requireThatTermReplacementsAreApplied() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + for (boolean specialToken : Arrays.asList(true, false)) { + for (TokenType type : TokenType.values()) { + if (!specialToken && !type.isIndexable()) { + continue; + } + assertAnnotations(expected, "foo", + newLinguistics(Arrays.asList(newToken("foo", "foo", type, specialToken)), + Collections.singletonMap("foo", "bar"))); + } + } + } + + @Test + public void requireThatExistingAnnotationsAreKept() { + SpanTree spanTree = new SpanTree(SpanTrees.LINGUISTICS); + spanTree.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz"))); + + StringFieldValue val = new StringFieldValue("foo"); + val.setSpanTree(spanTree); + + Linguistics linguistics = newLinguistics(Arrays.asList(newToken("foo", "bar", TokenType.ALPHABETIC, false)), + Collections.<String, String>emptyMap()); + new LinguisticsAnnotator(linguistics, CONFIG).annotate(val); + + assertTrue(new LinguisticsAnnotator(linguistics, CONFIG).annotate(val)); + assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS)); + } + + @Test + public void requireThatMaxTermOccurencesIsHonored() { + final String inputTerm = "foo"; + final String stemmedInputTerm = "bar"; // completely different from + // inputTerm for safer test + final String paddedInputTerm = inputTerm + " "; + final SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + final int inputTermOccurence = AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES * 2; + for (int i = 0; i < AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES; ++i) { + expected.spanList().span(i * paddedInputTerm.length(), inputTerm.length()) + .annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue(stemmedInputTerm))); + } + for (TokenType type : TokenType.values()) { + if (!type.isIndexable()) { + continue; + } + StringBuilder input = new StringBuilder(); + Token[] tokens = new Token[inputTermOccurence]; + for (int i = 0; i < inputTermOccurence; ++i) { + SimpleToken t = newToken(inputTerm, stemmedInputTerm, type); + t.setOffset(i * paddedInputTerm.length()); + tokens[i] = t; + input.append(paddedInputTerm); + } + assertAnnotations(expected, input.toString(), tokens); + } + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static SimpleToken newToken(String orig, String stem, TokenType type) { + return newToken(orig, stem, type, false); + } + + private static SimpleToken newToken(String orig, String stem, TokenType type, boolean specialToken) { + return new SimpleToken(orig).setTokenString(stem) + .setType(type) + .setSpecialToken(specialToken); + } + + private static void assertAnnotations(SpanTree expected, String value, Token... tokens) { + assertAnnotations(expected, value, newLinguistics(Arrays.asList(tokens), Collections.<String, String>emptyMap())); + } + + private static void assertAnnotations(SpanTree expected, String str, Linguistics linguistics) { + StringFieldValue val = new StringFieldValue(str); + assertEquals(expected != null, new LinguisticsAnnotator(linguistics, CONFIG).annotate(val)); + assertEquals(expected, val.getSpanTree(SpanTrees.LINGUISTICS)); + } + + private static Linguistics newLinguistics(List<? extends Token> tokens, Map<String, String> replacementTerms) { + Linguistics linguistics = Mockito.mock(Linguistics.class); + Mockito.when(linguistics.getTokenizer()).thenReturn(new MyTokenizer(tokens, replacementTerms)); + return linguistics; + } + + private static class MyTokenizer implements Tokenizer { + + final List<Token> tokens; + final Map<String, String> replacementTerms; + + public MyTokenizer(List<? extends Token> tokens, Map<String, String> replacementTerms) { + this.tokens = new ArrayList<>(tokens); + this.replacementTerms = replacementTerms; + } + + @Override + public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { + return tokens; + } + + @Override + public String getReplacementTerm(String term) { + String replacement = replacementTerms.get(term); + return replacement != null ? replacement : term; + } + } +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java index a81fb3ff152..1cffaeffb67 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java @@ -302,4 +302,4 @@ public class HeaderFields implements Map<String, List<String>> { throw new UnsupportedOperationException(); } } -}
\ No newline at end of file +} diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSetSelector.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSetSelector.java index a480d3968c9..129c7ea8c31 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSetSelector.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingSetSelector.java @@ -30,4 +30,4 @@ public interface BindingSetSelector { * @return The name of selected BindingSet. */ public String select(URI uri); -}
\ No newline at end of file +} diff --git a/jrt/pom.xml b/jrt/pom.xml index 6592cd84638..08ef9693750 100644 --- a/jrt/pom.xml +++ b/jrt/pom.xml @@ -52,7 +52,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/juniper/src/testproject.el b/juniper/src/testproject.el index 56cc68fe61d..8b52d77d651 100644 --- a/juniper/src/testproject.el +++ b/juniper/src/testproject.el @@ -6,7 +6,7 @@ ;; the class(es) you want to test. ;; $Revision: 1.2 $ $Date: 2003-02-27 12:32:24 $ -;; Author: Nils Sandøy <nils.sandoy@fast.no> +;; Author: Nils Sandøy <nils.sandoy@fast.no> ;; Just a message to show that this file is beeing read. Look for this ;; in the *Messages* buffer. diff --git a/libmlr/pom.xml b/libmlr/pom.xml index 53ac04ffcc7..46f2e5447fa 100644 --- a/libmlr/pom.xml +++ b/libmlr/pom.xml @@ -39,7 +39,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/linguistics/src/test/java/com/yahoo/language/simple/SimpleNormalizerTestCase.java b/linguistics/src/test/java/com/yahoo/language/simple/SimpleNormalizerTestCase.java index 9c9c8b8fcc5..bbff3ea97e1 100644 --- a/linguistics/src/test/java/com/yahoo/language/simple/SimpleNormalizerTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/simple/SimpleNormalizerTestCase.java @@ -31,4 +31,4 @@ public class SimpleNormalizerTestCase { assertEquals(expectedNormalForm, NORMALIZER.normalize(input)); } -}
\ No newline at end of file +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/TraceLevel.java b/messagebus/src/main/java/com/yahoo/messagebus/TraceLevel.java index 07b91a3280f..fb92ccf5f98 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/TraceLevel.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/TraceLevel.java @@ -27,4 +27,4 @@ public final class TraceLevel { * Traces information about which internal components are processing a routable. */ public static final int COMPONENT = 6; -}
\ No newline at end of file +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/package-info.java b/messagebus/src/main/java/com/yahoo/messagebus/network/package-info.java index 2fc632f82ba..429f419b3a8 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/package-info.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/package-info.java @@ -5,4 +5,4 @@ @ExportPackage package com.yahoo.messagebus.network; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java index ac7f349acd7..f23bd3b16a4 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java @@ -5,4 +5,4 @@ @ExportPackage package com.yahoo.messagebus.network.rpc; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/PolicyDirective.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/PolicyDirective.java index 7dd1966cb7b..e5009809925 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/PolicyDirective.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/PolicyDirective.java @@ -80,4 +80,4 @@ public class PolicyDirective implements HopDirective { result = 31 * result + (param != null ? param.hashCode() : 0); return result; } -}
\ No newline at end of file +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java index d7105a92f16..2a469d64857 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingContext.java @@ -34,6 +34,11 @@ public class RoutingContext { this.directive = directive; } + public String toString() { + return "node : " + node + ", directive: " + directive + ", errors: " + consumableErrors + + ", selectOnRetry: " + selectOnRetry + " context: " + context; + } + /** * <p>Returns whether or not this hop has any configured recipients.</p> * diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 26c103e7877..72cc6ef41fd 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -492,7 +492,7 @@ public class RoutingNode implements ReplyHandler { return false; } - private String exceptionMessageWithTrace(Exception e) { + private static String exceptionMessageWithTrace(Exception e) { StringWriter sw = new StringWriter(); try (PrintWriter pw = new PrintWriter(sw)) { e.printStackTrace(pw); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/package-info.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/package-info.java index bedb9d96f26..00cb21c769f 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/package-info.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/package-info.java @@ -5,4 +5,4 @@ @ExportPackage package com.yahoo.messagebus.routing; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServicePoolTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServicePoolTestCase.java index a35f98183da..2493e6dbf88 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServicePoolTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServicePoolTestCase.java @@ -54,4 +54,4 @@ public class ServicePoolTestCase extends TestCase { slobrok.stop(); } -}
\ No newline at end of file +} diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java index 821080524af..a77b1f28cc4 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java @@ -109,4 +109,4 @@ public class TargetPoolTestCase extends junit.framework.TestCase { return millis; } } -}
\ No newline at end of file +} diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java index ea217af5b9a..f9797be9db0 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java @@ -121,6 +121,10 @@ public class RoutingContextTestCase extends junit.framework.TestCase { } } + public void testToString() { + assertEquals("node : null, directive: 1, errors: [], selectOnRetry: true context: null", new RoutingContext(null, 1).toString()); + } + public void testConstRoute() { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("DocumentRouteSelector", diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java index 4a9ea225bdd..f9d3473bbf9 100644 --- a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java +++ b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java @@ -29,4 +29,4 @@ public abstract class MetricVisitor { public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) { return true; } -}
\ No newline at end of file +} diff --git a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java index 6335d329836..cc6d693ce1f 100644 --- a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java +++ b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java @@ -235,4 +235,4 @@ public class SumMetric extends Metric @Override public void reset() {} -}
\ No newline at end of file +} diff --git a/node-admin/configserver-app/hosted-vespa/http-server.xml b/node-admin/configserver-app/hosted-vespa/http-server.xml index f77d392de97..6b2b2099198 100644 --- a/node-admin/configserver-app/hosted-vespa/http-server.xml +++ b/node-admin/configserver-app/hosted-vespa/http-server.xml @@ -1 +1 @@ -<server port="4080" id="configserver-real" />
\ No newline at end of file +<server port="4080" id="configserver-real" /> diff --git a/node-admin/include/nodectl-instance.sh b/node-admin/include/nodectl-instance.sh index b3d3dbea440..a1018d6ba12 100644 --- a/node-admin/include/nodectl-instance.sh +++ b/node-admin/include/nodectl-instance.sh @@ -188,4 +188,4 @@ main() { fi } -main "$@"
\ No newline at end of file +main "$@" diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 9922fd489fe..9b27d933a4a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -420,4 +420,4 @@ public class StorageMaintainer { nextHandleOldCoredumpsAt = clock.instant().minus(Duration.ofDays(1)); } } -}
\ No newline at end of file +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java index 5e5bd002950..dcd87659cfa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.vespa.hosted.node.admin.restapi; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index a4556b5c8bc..7a3461679af 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -121,4 +121,4 @@ public class StorageMaintainerTest { private static void writeNBytesToFile(File file, int nBytes) throws IOException { Files.write(file.toPath(), new byte[nBytes]); } -}
\ No newline at end of file +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java index 7d6a95d8dc8..4e40b003ec8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java @@ -136,4 +136,4 @@ public class ConfigServerHttpRequestExecutorTest { assertTrue("log does not contain expected entries:" + logString, (logString.equals("GET http://host1:666/path ") || logString.equals("GET http://host2:666/path "))); } -}
\ No newline at end of file +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java index d89a4761b5d..f1981ced48e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirer.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.Flavor; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareChecker; import java.time.Duration; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -35,7 +37,7 @@ public class NodeRetirer extends Maintainer { public static final FlavorSpareChecker.SpareNodesPolicy SPARE_NODES_POLICY = flavorSpareCount -> flavorSpareCount.getNumReadyAmongReplacees() > 2; - private static final long MAX_SIMULTANEOUS_RETIRES_PER_APPLICATION = 1; + private static final long MAX_SIMULTANEOUS_RETIRES_PER_CLUSTER = 1; private static final Logger log = Logger.getLogger(NodeRetirer.class.getName()); private final Deployer deployer; @@ -115,16 +117,19 @@ public class NodeRetirer extends Maintainer { Map<Deployment, Set<Node>> nodesToRetireByDeployment = new HashMap<>(); for (ApplicationId applicationId : activeApplications) { List<Node> applicationNodes = getNodesBelongingToApplication(allNodes, applicationId); - Set<Node> retireableNodes = getRetireableNodesForApplication(applicationNodes); - long numNodesAllowedToRetire = getNumberNodesAllowToRetireForApplication(applicationNodes, MAX_SIMULTANEOUS_RETIRES_PER_APPLICATION); - if (retireableNodes.isEmpty() || numNodesAllowedToRetire == 0) continue; + Map<ClusterSpec, Set<Node>> retireableNodesByCluster = getRetireableNodesForApplication(applicationNodes).stream() + .collect(Collectors.groupingBy( + node -> node.allocation().get().membership().cluster(), + Collectors.toSet())); + if (retireableNodesByCluster.isEmpty()) continue; Optional<Deployment> deployment = deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30)); if ( ! deployment.isPresent()) continue; // this will be done at another config server - Set<Node> replaceableNodes = retireableNodes.stream() - .filter(node -> flavorSpareChecker.canRetireAllocatedNodeWithFlavor(node.flavor())) - .limit(numNodesAllowedToRetire) + Set<Node> replaceableNodes = retireableNodesByCluster.values().stream() + .flatMap(nodes -> nodes.stream() + .filter(node -> flavorSpareChecker.canRetireAllocatedNodeWithFlavor(node.flavor())) + .limit(getNumberNodesAllowToRetireForCluster(nodes, MAX_SIMULTANEOUS_RETIRES_PER_CLUSTER))) .collect(Collectors.toSet()); if (! replaceableNodes.isEmpty()) nodesToRetireByDeployment.put(deployment.get(), replaceableNodes); } @@ -162,7 +167,7 @@ public class NodeRetirer extends Maintainer { })); } - private List<Node> getNodesBelongingToApplication(List<Node> allNodes, ApplicationId applicationId) { + private List<Node> getNodesBelongingToApplication(Collection<Node> allNodes, ApplicationId applicationId) { return allNodes.stream() .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(applicationId)) @@ -172,7 +177,7 @@ public class NodeRetirer extends Maintainer { /** * Returns a list of ApplicationIds sorted by number of active nodes the application has allocated to it */ - List<ApplicationId> getActiveApplicationIds(List<Node> nodes) { + List<ApplicationId> getActiveApplicationIds(Collection<Node> nodes) { return nodes.stream() .filter(node -> node.state() == Node.State.active) .collect(Collectors.groupingBy( @@ -188,7 +193,7 @@ public class NodeRetirer extends Maintainer { * @param applicationNodes All the nodes allocated to an application * @return Set of nodes that all should eventually be retired */ - Set<Node> getRetireableNodesForApplication(List<Node> applicationNodes) { + Set<Node> getRetireableNodesForApplication(Collection<Node> applicationNodes) { return applicationNodes.stream() .filter(node -> node.state() == Node.State.active) .filter(node -> !node.status().wantToRetire()) @@ -197,18 +202,18 @@ public class NodeRetirer extends Maintainer { } /** - * @param applicationNodes All the nodes allocated to an application + * @param clusterNodes All the nodes allocated to an application belonging to a single cluster * @return number of nodes we can safely start retiring */ - long getNumberNodesAllowToRetireForApplication(List<Node> applicationNodes, long maxSimultaneousRetires) { - long numNodesInWantToRetire = applicationNodes.stream() + long getNumberNodesAllowToRetireForCluster(Collection<Node> clusterNodes, long maxSimultaneousRetires) { + long numNodesInWantToRetire = clusterNodes.stream() .filter(node -> node.status().wantToRetire()) .filter(node -> node.state() != Node.State.parked) .count(); return Math.max(0, maxSimultaneousRetires - numNodesInWantToRetire); } - private Map<Flavor, Map<Node.State, Long>> getNumberOfNodesByFlavorByNodeState(List<Node> allNodes) { + private Map<Flavor, Map<Node.State, Long>> getNumberOfNodesByFlavorByNodeState(Collection<Node> allNodes) { return allNodes.stream() .collect(Collectors.groupingBy( Node::flavor, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java index be45c0df8e0..261cc179438 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/package-info.java @@ -4,4 +4,4 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.osgi.annotation.ExportPackage; -/** The node repository controls and allocates the nodes available in a hosted Vespa zone */
\ No newline at end of file +/** The node repository controls and allocates the nodes available in a hosted Vespa zone */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java index d7cd8bb1e19..5454a7ca1c4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerCapacityConstraints.java @@ -98,4 +98,4 @@ public class DockerCapacityConstraints { return new NodeAllocation(appId, cluster, new NodeSpec.CountNodeSpec(count, flavor), new MutableInteger(0), Clock.systemUTC()); } -}
\ No newline at end of file +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index e149b952fa7..0d9a75e6140 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -247,4 +247,4 @@ public class NodeAllocation { return Comparator.comparing((Node n) -> n.allocation().get().membership().index()); } -}
\ No newline at end of file +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java index 5859a3a5bee..0ad4768c157 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java @@ -4,4 +4,4 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.osgi.annotation.ExportPackage; -/** Implements the provisioning API to perform node provisioning from a node repository */
\ No newline at end of file +/** Implements the provisioning API to perform node provisioning from a node repository */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index e46797240f2..d39b61f367d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * @author bratseth @@ -87,11 +88,39 @@ public class MockDeployer implements Deployer { public static class ApplicationContext { private final ApplicationId id; + private final List<ClusterContext> clusterContexts; + + public ApplicationContext(ApplicationId id, List<ClusterContext> clusterContexts) { + this.id = id; + this.clusterContexts = clusterContexts; + } + + public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) { + this(id, Collections.singletonList(new ClusterContext(id, cluster, capacity, groups))); + } + + public ApplicationId id() { return id; } + + /** Returns list of cluster specs of this application. */ + public List<ClusterContext> clusterContexts() { return clusterContexts; } + + private List<HostSpec> prepare(NodeRepositoryProvisioner provisioner) { + return clusterContexts.stream() + .map(clusterContext -> clusterContext.prepare(provisioner)) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + } + + public static class ClusterContext { + + private final ApplicationId id; private final ClusterSpec cluster; private final Capacity capacity; private final int groups; - public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) { + public ClusterContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) { this.id = id; this.cluster = cluster; this.capacity = capacity; @@ -100,7 +129,6 @@ public class MockDeployer implements Deployer { public ApplicationId id() { return id; } - /** Returns the spec of the cluster of this application. Only a single cluster per application is supported */ public ClusterSpec cluster() { return cluster; } private List<HostSpec> prepare(NodeRepositoryProvisioner provisioner) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java index 983f81be126..e6a50cbbc64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ServiceMonitorStub.java @@ -77,7 +77,7 @@ public class ServiceMonitorStub implements ServiceMonitor { getHostStatus(node.hostname()))); } Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = new HashSet<>(); - serviceClusters.add(new ServiceCluster<>(new ClusterId(app.getValue().cluster().id().value()), + serviceClusters.add(new ServiceCluster<>(new ClusterId(app.getValue().clusterContexts().get(0).cluster().id().value()), new ServiceType("serviceType"), serviceInstances)); TenantId tenantId = new TenantId(app.getKey().tenant().value()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java index f71d9b75f78..b19d9360e86 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTest.java @@ -34,24 +34,25 @@ public class NodeRetirerTest { retirer = tester.makeNodeRetirer(policy); tester.createReadyNodesByFlavor(21, 42, 27, 15, 8); - tester.deployApp("vespa", "calendar", 3, 7); - tester.deployApp("vespa", "notes", 0, 3); - tester.deployApp("sports", "results", 0, 6); - tester.deployApp("search", "images", 3, 4); - tester.deployApp("search", "videos", 2, 2); + tester.deployApp("vespa", "calendar", new int[]{3}, new int[]{7}); + tester.deployApp("vespa", "notes", new int[]{0}, new int[]{3}); + tester.deployApp("sports", "results", new int[]{0}, new int[]{6}); + tester.deployApp("search", "images", new int[]{3}, new int[]{4}); + tester.deployApp("search", "videos", new int[]{2}, new int[]{2}); + tester.deployApp("tester", "my-app", new int[]{1, 2}, new int[]{4, 6}); } @Test public void testRetireUnallocated() { - tester.assertCountsForStateByFlavor(Node.State.ready, 12, 42, 25, 4, 8); - tester.setNumberAllowedUnallocatedRetirementsPerFlavor(6, 30, 20, 2, 4); + tester.assertCountsForStateByFlavor(Node.State.ready, 12, 38, 19, 4, 8); + tester.setNumberAllowedUnallocatedRetirementsPerFlavor(6, 30, 15, 2, 4); assertFalse(retirer.retireUnallocated()); - tester.assertCountsForStateByFlavor(Node.State.parked, 6, 30, 20, 2, 4); + tester.assertCountsForStateByFlavor(Node.State.parked, 6, 30, 15, 2, 4); - tester.assertCountsForStateByFlavor(Node.State.ready, 6, 12, 5, 2, 4); + tester.assertCountsForStateByFlavor(Node.State.ready, 6, 8, 4, 2, 4); tester.setNumberAllowedUnallocatedRetirementsPerFlavor(10, 20, 5, 5, 4); assertTrue(retirer.retireUnallocated()); - tester.assertCountsForStateByFlavor(Node.State.parked, 12, 42, 25, 4, 8); + tester.assertCountsForStateByFlavor(Node.State.parked, 12, 38, 19, 4, 8); tester.nodeRepository.getNodes().forEach(node -> assertEquals(node.status().wantToDeprovision(), node.state() == Node.State.parked)); @@ -63,25 +64,27 @@ public class NodeRetirerTest { tester.nodeRepository.getNodes(Node.State.ready) .forEach(node -> tester.nodeRepository.write(node.withIpAddresses(Collections.singleton("::2")))); - tester.assertCountsForStateByFlavor(Node.State.active, 9, -1, 2, 11, -1); + tester.assertCountsForStateByFlavor(Node.State.active, 9, 4, 8, 11, -1); - tester.setNumberAllowedAllocatedRetirementsPerFlavor(3, 2, 3, 2); + tester.setNumberAllowedAllocatedRetirementsPerFlavor(3, 2, 4, 2); retirer.retireAllocated(); - tester.assertParkedCountsByApplication(-1, -1, -1, -1, -1); // Nodes should be in retired, but not yet parked + tester.assertParkedCountsByApplication(-1, -1, -1, -1, -1, -1); // Nodes should be in retired, but not yet parked tester.iterateMaintainers(); - tester.assertParkedCountsByApplication(1, 1, 1, 1, 1); + tester.assertParkedCountsByApplication(1, 1, 1, 1, 1, 2); - // We can only retire 1 more of flavor 0 and 1 more of flavor 2, app 3 is the largest that is on flavor 0 - // and app 5 is the only one on flavor 2 + // We can retire 1 more of flavor-0, 1 more of flavor-1, 2 more of flavor-2: + // app 6 has the most nodes, so it gets to retire flavor-1 and flavor-2 + // app 3 is the largest that is on flavor-0, so it gets the last node + // app 5 is gets the last node with flavor-2 retirer.retireAllocated(); tester.iterateMaintainers(); - tester.assertParkedCountsByApplication(1, 1, 2, 1, 2); + tester.assertParkedCountsByApplication(1, 1, 2, 1, 2, 4); // No more retirements are possible retirer.retireAllocated(); tester.iterateMaintainers(); - tester.assertParkedCountsByApplication(1, 1, 2, 1, 2); + tester.assertParkedCountsByApplication(1, 1, 2, 1, 2, 4); tester.nodeRepository.getNodes().forEach(node -> assertEquals(node.status().wantToDeprovision(), node.state() == Node.State.parked)); @@ -90,7 +93,7 @@ public class NodeRetirerTest { @Test public void testGetActiveApplicationIds() { List<String> expectedOrder = Arrays.asList( - "vespa.calendar", "sports.results", "search.images", "vespa.notes", "search.videos"); + "tester.my-app", "vespa.calendar", "sports.results", "search.images", "vespa.notes", "search.videos"); List<String> actualOrder = retirer.getActiveApplicationIds(tester.nodeRepository.getNodes()).stream() .map(applicationId -> applicationId.toShortString().replace(":default", "")) .collect(Collectors.toList()); @@ -121,21 +124,21 @@ public class NodeRetirerTest { } @Test - public void testGetNumberNodesAllowToRetireForApplication() { + public void testGetNumberNodesAllowToRetireForCluster() { ApplicationId app = new ApplicationId.Builder().tenant("vespa").applicationName("calendar").build(); - long actualAllActive = retirer.getNumberNodesAllowToRetireForApplication(tester.nodeRepository.getNodes(app), 2); + long actualAllActive = retirer.getNumberNodesAllowToRetireForCluster(tester.nodeRepository.getNodes(app), 2); assertEquals(2, actualAllActive); // Lets put 3 random nodes in wantToRetire List<Node> nodesToRetire = tester.nodeRepository.getNodes(app).stream().limit(3).collect(Collectors.toList()); nodesToRetire.forEach(node -> tester.nodeRepository.write(node.with(node.status().withWantToRetire(true)))); - long actualOneWantToRetire = retirer.getNumberNodesAllowToRetireForApplication(tester.nodeRepository.getNodes(app), 2); + long actualOneWantToRetire = retirer.getNumberNodesAllowToRetireForCluster(tester.nodeRepository.getNodes(app), 2); assertEquals(0, actualOneWantToRetire); // Now 2 of those finish retiring and go to parked nodesToRetire.stream().limit(2).forEach(node -> tester.nodeRepository.park(node.hostname(), Agent.system, "Parked for unit testing")); - long actualOneRetired = retirer.getNumberNodesAllowToRetireForApplication(tester.nodeRepository.getNodes(app), 2); + long actualOneRetired = retirer.getNumberNodesAllowToRetireForCluster(tester.nodeRepository.getNodes(app), 2); assertEquals(1, actualOneRetired); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java index c18ebd67d62..54690cda6a5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRetirerTester.java @@ -8,16 +8,13 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy; @@ -55,11 +52,9 @@ public class NodeRetirerTester { public final ManualClock clock = new ManualClock(); public final NodeRepository nodeRepository; private final FlavorSpareChecker flavorSpareChecker = mock(FlavorSpareChecker.class); - private final Curator curator = new MockCurator(); private final MockDeployer deployer; private final JobControl jobControl; private final List<Flavor> flavors; - private final NodeRepositoryProvisioner provisioner; // Use LinkedHashMap to keep order in which applications were deployed private final Map<ApplicationId, MockDeployer.ApplicationContext> apps = new LinkedHashMap<>(); @@ -69,10 +64,11 @@ public class NodeRetirerTester { private int nextNodeId = 0; NodeRetirerTester(NodeFlavors nodeFlavors) { + Curator curator = new MockCurator(); nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), new DockerImage("docker-registry.domain.tld:8080/dist/vespa")); jobControl = new JobControl(nodeRepository.database()); - provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, nodeFlavors, zone); deployer = new MockDeployer(provisioner, apps); flavors = nodeFlavors.getFlavors().stream().sorted(Comparator.comparing(Flavor::name)).collect(Collectors.toList()); } @@ -97,15 +93,19 @@ public class NodeRetirerTester { nodeRepository.setReady(nodes); } - void deployApp(String tenantName, String applicationName, int flavorId, int numNodes) { - Flavor flavor = flavors.get(flavorId); + void deployApp(String tenantName, String applicationName, int[] flavorIds, int[] numNodes) { + final ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); + final List<MockDeployer.ClusterContext> clusterContexts = new ArrayList<>(); - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.99")); - Capacity capacity = Capacity.fromNodeCount(numNodes, flavor.name()); - apps.put(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, capacity, 1)); + for (int i = 0; i < flavorIds.length; i++) { + Flavor flavor = flavors.get(flavorIds[i]); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("cluster-" + i), Version.fromString("6.99")); + Capacity capacity = Capacity.fromNodeCount(numNodes[i], flavor.name()); + clusterContexts.add(new MockDeployer.ClusterContext(applicationId, cluster, capacity, 1)); + } - activate(applicationId, cluster, capacity); + apps.put(applicationId, new MockDeployer.ApplicationContext(applicationId, clusterContexts)); + deployer.deployFromLocalActive(applicationId, Duration.ZERO).get().activate(); } void iterateMaintainers() { @@ -121,13 +121,6 @@ public class NodeRetirerTester { inactiveExpirer.maintain(); } - private void activate(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, capacity, 1, null); - NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, applicationId, hosts); - transaction.commit(); - } - void setNumberAllowedUnallocatedRetirementsPerFlavor(int... numAllowed) { for (int i = 0; i < numAllowed.length; i++) { Boolean[] responses = new Boolean[numAllowed[i]]; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java index ef592974756..2a0b9aa5067 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java @@ -150,4 +150,4 @@ public class AllocationSimulator { simulator.addCluster("App5 : 3 * d-3 nodes", 3, simulator.flavors.getFlavorOrThrow("d-3"), "App5"); simulator.addCluster("App6 : 5 * d-2 nodes", 5, simulator.flavors.getFlavorOrThrow("d-2"), "App6"); } -}
\ No newline at end of file +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java index ba9a7f2924f..abb283a0cf3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationVisualizer.java @@ -140,4 +140,4 @@ public class AllocationVisualizer extends JPanel { } } } -}
\ No newline at end of file +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index 082002b7cf2..7e16755ccd4 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -130,4 +130,4 @@ public class ClusterApiImplTest { assertEquals(Optional.of(hostName1), clusterApi.storageNodeInGroup().map(storageNode -> storageNode.hostName())); assertEquals(Optional.of(hostName1), clusterApi.upStorageNodeInGroup().map(storageNode -> storageNode.hostName())); } -}
\ No newline at end of file +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java index 56a9e6bf147..35c0629dfcd 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/NodeGroupTest.java @@ -30,4 +30,4 @@ public class NodeGroupTest { assertEquals(Arrays.asList(hostName1, hostName2, hostName3), nodeGroup.getHostNames()); assertEquals("host1,host2,host3", nodeGroup.toCommaSeparatedString()); } -}
\ No newline at end of file +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index e6e60900906..6f59f0c5973 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -110,4 +110,4 @@ public class HostedVespaClusterPolicyTest { } } } -}
\ No newline at end of file +} @@ -121,7 +121,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> + <version>2.5.2</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/sample-apps/basic-search-java/src/main/java/com/yahoo/example/StringData.java b/sample-apps/basic-search-java/src/main/java/com/yahoo/example/StringData.java index 0bff013ab78..b177881035a 100644 --- a/sample-apps/basic-search-java/src/main/java/com/yahoo/example/StringData.java +++ b/sample-apps/basic-search-java/src/main/java/com/yahoo/example/StringData.java @@ -17,4 +17,4 @@ public class StringData extends AbstractData { public String toString() { return string; } -}
\ No newline at end of file +} diff --git a/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ApplicationMain.java b/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ApplicationMain.java index da85e486602..638df584f92 100644 --- a/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ApplicationMain.java +++ b/sample-apps/basic-search-java/src/test/java/com/yahoo/example/ApplicationMain.java @@ -24,4 +24,4 @@ public class ApplicationMain { Thread.sleep(Long.MAX_VALUE); } } -}
\ No newline at end of file +} diff --git a/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig index 59b173e16f4..347958bfb84 100644 --- a/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig +++ b/sample-apps/blog-recommendation/src/pig/feed_content_vespa.pig @@ -68,4 +68,4 @@ data_for_feed_json = FOREACH data_for_feed GENERATE VespaPutOperationDoc(*); -- STORE data_for_feed_json_sample INTO 'blog-sample'; -- Store into Vespa -STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage();
\ No newline at end of file +STORE data_for_feed_json INTO '$ENDPOINT' USING VespaStorage(); diff --git a/sample-apps/blog-recommendation/src/spark/expected_percentile.scala b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala index 986a3eb79f4..7236f8e8150 100644 --- a/sample-apps/blog-recommendation/src/spark/expected_percentile.scala +++ b/sample-apps/blog-recommendation/src/spark/expected_percentile.scala @@ -36,4 +36,4 @@ val expected_percentile = joined_data. count($"post_id").as("number_read")). withColumn("expected_percentile", $"sum_percentile" / $"number_read") -expected_percentile.show()
\ No newline at end of file +expected_percentile.show() diff --git a/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala index 2fc67734386..d21f96028bc 100644 --- a/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala +++ b/sample-apps/blog-recommendation/src/spark/train_test_set_division.scala @@ -42,4 +42,4 @@ test_set_exploded = test_set_exploded.unionAll(test_set_null.select("post_id").w // Write to disk training_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(training_file_path) -test_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(test_file_path)
\ No newline at end of file +test_set_exploded.rdd.map(x => x(0) + "\t" + x(1)).saveAsTextFile(test_file_path) diff --git a/sample-apps/blog-tutorial-shared/src/main/pig/tutorial_compute_metric.pig b/sample-apps/blog-tutorial-shared/src/main/pig/tutorial_compute_metric.pig index 50b4bc19967..7bde9b5c2b7 100644 --- a/sample-apps/blog-tutorial-shared/src/main/pig/tutorial_compute_metric.pig +++ b/sample-apps/blog-tutorial-shared/src/main/pig/tutorial_compute_metric.pig @@ -39,4 +39,4 @@ grouped_data = FOREACH grouped_data COUNT(joined_data.post_id) AS number_read, (double)SUM(joined_data.percentile)/(double)COUNT(joined_data.post_id) AS expected_percentile; -STORE grouped_data INTO '$OUTPUT';
\ No newline at end of file +STORE grouped_data INTO '$OUTPUT'; diff --git a/sample-apps/blog-tutorial-shared/src/test/python/parse-unittest.py b/sample-apps/blog-tutorial-shared/src/test/python/parse-unittest.py index 9579f619aed..d1fc725597d 100644 --- a/sample-apps/blog-tutorial-shared/src/test/python/parse-unittest.py +++ b/sample-apps/blog-tutorial-shared/src/test/python/parse-unittest.py @@ -176,4 +176,4 @@ class KaggleRawDataParserTest(unittest.TestCase): self.assertEqual(json.loads(output_array[i]), compare_with[i]) if __name__ == '__main__': - unittest.main()
\ No newline at end of file + unittest.main() diff --git a/sample-apps/blog-tutorial-shared/src/test/scala/com/yahoo/example/blog/BlogRecommendationAppTest.scala b/sample-apps/blog-tutorial-shared/src/test/scala/com/yahoo/example/blog/BlogRecommendationAppTest.scala index 6f79d752fa1..7ae8b8a8b06 100644 --- a/sample-apps/blog-tutorial-shared/src/test/scala/com/yahoo/example/blog/BlogRecommendationAppTest.scala +++ b/sample-apps/blog-tutorial-shared/src/test/scala/com/yahoo/example/blog/BlogRecommendationAppTest.scala @@ -6,4 +6,4 @@ class BlogRecommendationAppTest extends FunSuite { test("CollaborativeFilteringApp writes user and item latent factors to output path") (pending) -}
\ No newline at end of file +} diff --git a/sample-apps/http-api-using-request-handlers-and-processors/pom.xml b/sample-apps/http-api-using-request-handlers-and-processors/pom.xml index 7ae8549c385..8141a0b91b1 100644 --- a/sample-apps/http-api-using-request-handlers-and-processors/pom.xml +++ b/sample-apps/http-api-using-request-handlers-and-processors/pom.xml @@ -6,7 +6,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.yahoo.demo</groupId> - <artifactId>sample-app</artifactId> + <artifactId>sample-app-http-api-processor</artifactId> <version>1.0.1</version> <packaging>container-plugin</packaging> <!-- Use Vespa packaging --> diff --git a/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/AnnotatingProcessor.java b/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/AnnotatingProcessor.java index e241d59f5d7..ec2080bad00 100644 --- a/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/AnnotatingProcessor.java +++ b/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/AnnotatingProcessor.java @@ -94,4 +94,4 @@ public class AnnotatingProcessor extends Processor { return smurfFound; } -}
\ No newline at end of file +} diff --git a/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/DataProcessor.java b/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/DataProcessor.java index 915beb38f5e..d7f074d398f 100644 --- a/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/DataProcessor.java +++ b/sample-apps/http-api-using-request-handlers-and-processors/src/main/java/com/yahoo/demo/DataProcessor.java @@ -84,4 +84,4 @@ public class DataProcessor extends Processor { return r; } -}
\ No newline at end of file +} diff --git a/sample-apps/http-api-using-request-handlers-and-processors/src/test/java/com/yahoo/example/ApplicationMain.java b/sample-apps/http-api-using-request-handlers-and-processors/src/test/java/com/yahoo/example/ApplicationMain.java index da85e486602..638df584f92 100644 --- a/sample-apps/http-api-using-request-handlers-and-processors/src/test/java/com/yahoo/example/ApplicationMain.java +++ b/sample-apps/http-api-using-request-handlers-and-processors/src/test/java/com/yahoo/example/ApplicationMain.java @@ -24,4 +24,4 @@ public class ApplicationMain { Thread.sleep(Long.MAX_VALUE); } } -}
\ No newline at end of file +} diff --git a/sample-apps/http-api-using-searcher/pom.xml b/sample-apps/http-api-using-searcher/pom.xml index 7ae8549c385..6dfee6026ee 100644 --- a/sample-apps/http-api-using-searcher/pom.xml +++ b/sample-apps/http-api-using-searcher/pom.xml @@ -6,7 +6,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.yahoo.demo</groupId> - <artifactId>sample-app</artifactId> + <artifactId>sample-app-http-api-searcher</artifactId> <version>1.0.1</version> <packaging>container-plugin</packaging> <!-- Use Vespa packaging --> diff --git a/sample-apps/http-api-using-searcher/src/test/java/com/yahoo/example/ApplicationMain.java b/sample-apps/http-api-using-searcher/src/test/java/com/yahoo/example/ApplicationMain.java index da85e486602..638df584f92 100644 --- a/sample-apps/http-api-using-searcher/src/test/java/com/yahoo/example/ApplicationMain.java +++ b/sample-apps/http-api-using-searcher/src/test/java/com/yahoo/example/ApplicationMain.java @@ -24,4 +24,4 @@ public class ApplicationMain { Thread.sleep(Long.MAX_VALUE); } } -}
\ No newline at end of file +} diff --git a/sample-apps/pom.xml b/sample-apps/pom.xml index 91d1b7c7580..d39e3d1e13f 100644 --- a/sample-apps/pom.xml +++ b/sample-apps/pom.xml @@ -13,9 +13,7 @@ <module>basic-search-java</module> <module>blog-recommendation</module> <module>boolean-search</module> -<!-- <module>http-api-using-request-handlers-and-processors</module> <module>http-api-using-searcher</module> ---> </modules> </project> diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h index 96c3d7f3470..bd88c5e5901 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h +++ b/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h @@ -2,10 +2,9 @@ #pragma once -#include <stddef.h> +#include <cstddef> -namespace search { -namespace attribute { +namespace search::attribute { class IAttributeVector; @@ -45,4 +44,3 @@ public: }; } -} diff --git a/searchcommon/src/vespa/searchcommon/common/range.h b/searchcommon/src/vespa/searchcommon/common/range.h index 5bcf2355eb9..f3e3156491a 100644 --- a/searchcommon/src/vespa/searchcommon/common/range.h +++ b/searchcommon/src/vespa/searchcommon/common/range.h @@ -5,10 +5,9 @@ #pragma once #include <limits> -#include <stdint.h> +#include <cstdint> -namespace search -{ +namespace search { template <typename T> class Range { @@ -30,4 +29,3 @@ private: using Int64Range = Range<int64_t>; } // namespace search - diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp index 2bd652b59c7..da7f8830ee8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp @@ -36,4 +36,4 @@ DocumentSubDbInitializer::run() (void) future.get(); } -} // namespace proton
\ No newline at end of file +} // namespace proton diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp index d2788ed558a..9e59ca1e24d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp @@ -109,4 +109,4 @@ void DiskIndexCleaner::removeOldIndexes( removeOld(base_dir, indexes, active_indexes); } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java index c7ff00a9e78..0a7b36cfd71 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/DivideFunctionNode.java @@ -20,4 +20,4 @@ public class DivideFunctionNode extends NumericFunctionNode { protected void onArgument(final ResultNode arg, ResultNode result) { ((NumericResultNode)result).divide(arg); } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java index 4a636bb8a3f..804b6a52c5b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java @@ -146,4 +146,4 @@ public class Int16ResultNode extends NumericResultNode { public void set(ResultNode rhs) { value = (short)rhs.getInteger(); } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java index 0316ee2d9c3..9a03df015b8 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java @@ -146,4 +146,4 @@ public class Int32ResultNode extends NumericResultNode { public void set(ResultNode rhs) { value = (int)rhs.getInteger(); } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java index 0bbc3414c2e..d21473b9be8 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java @@ -146,4 +146,4 @@ public class Int8ResultNode extends NumericResultNode { public void set(ResultNode rhs) { value = (byte)rhs.getInteger(); } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java index 102c598402a..c7897afd514 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/NumElemFunctionNode.java @@ -47,4 +47,4 @@ public class NumElemFunctionNode extends UnaryFunctionNode { protected boolean equalsUnaryFunction(UnaryFunctionNode obj) { return true; } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/mlr/ga/KeyboardChecker.java b/searchlib/src/main/java/com/yahoo/searchlib/mlr/ga/KeyboardChecker.java index 091990ffdc5..6d47b2dfd7a 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/mlr/ga/KeyboardChecker.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/mlr/ga/KeyboardChecker.java @@ -47,4 +47,4 @@ public class KeyboardChecker { } } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/SegmentStartPoint.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/SegmentStartPoint.java index c7331a0b5b6..15e458dea70 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/SegmentStartPoint.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/SegmentStartPoint.java @@ -142,4 +142,4 @@ final class SegmentStartPoint { " (" + (open ? "open" : "closed") + ")"; } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 86ac53a1e44..7e5033844a3 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -1,139 +1,139 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.rankingexpression;
-
-import com.google.common.collect.ImmutableList;
-import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
-import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
-import com.yahoo.text.Utf8;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.*;
-
-/**
- * <p>A function defined by a ranking expression</p>
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- * @author bratseth
- */
-public class ExpressionFunction {
-
- private final String name;
- private final ImmutableList<String> arguments;
- private final RankingExpression body;
-
- /**
- * <p>Constructs a new function</p>
- *
- * @param name the name of this function
- * @param arguments its argument names
- * @param body the ranking expression that defines this function
- */
- public ExpressionFunction(String name, List<String> arguments, RankingExpression body) {
- this.name = name;
- this.arguments = arguments==null ? ImmutableList.of() : ImmutableList.copyOf(arguments);
- this.body = body;
- }
-
- public String getName() { return name; }
-
- /** Returns an immutable list of the arguments of this */
- public List<String> arguments() { return arguments; }
-
- public RankingExpression getBody() { return body; }
-
- /**
- * <p>Create and return an instance of this function based on the given
- * arguments. If function calls are nested, this call might produce
- * additional scripts.</p>
- *
- * @param context the context used to expand this
- * @param arguments the arguments to instantiate on.
- * @param path the expansion path leading to this.
- * @return the script function instance created.
- */
- public Instance expand(SerializationContext context, List<ExpressionNode> arguments, Deque<String> path) {
- Map<String, String> argumentBindings = new HashMap<>();
- for (int i = 0; i < this.arguments.size() && i < arguments.size(); ++i) {
- argumentBindings.put(this.arguments.get(i), arguments.get(i).toString(context, path, null));
- }
- return new Instance(toSymbol(argumentBindings), body.getRoot().toString(context.createBinding(argumentBindings), path, null));
- }
-
- /**
- * Returns a symbolic string that represents this function with a given
- * list of arguments. The arguments are mangled by hashing the string
- * representation of the argument expressions, so we might need to revisit
- * this if we start seeing collisions.
- *
- * @param argumentBindings the bound arguments to include in the symbolic name.
- * @return the symbolic name for an instance of this function
- */
- private String toSymbol(Map<String, String> argumentBindings) {
- if (argumentBindings.isEmpty()) return name;
-
- StringBuilder ret = new StringBuilder();
- ret.append(name).append("@");
- for (Map.Entry<String,String> argumentBinding : argumentBindings.entrySet()) {
- ret.append(Long.toHexString(symbolCode(argumentBinding.getKey() + "=" + argumentBinding.getValue())));
- ret.append(".");
- }
- if (ret.toString().endsWith("."))
- ret.setLength(ret.length()-1);
- return ret.toString();
- }
-
-
- /**
- * <p>Returns a more unique hash code than what Java's own {@link
- * String#hashCode()} method would produce.</p>
- *
- * @param str The string to hash.
- * @return A 64 bit long hash code.
- */
- private static long symbolCode(String str) {
- try {
- MessageDigest md = java.security.MessageDigest.getInstance("SHA-1");
- byte[] buf = md.digest(Utf8.toBytes(str));
- if (buf.length >= 8) {
- long ret = 0;
- for (int i = 0; i < 8; ++i) {
- ret = (ret << 8) + (buf[i] & 0xff);
- }
- return ret;
- }
- } catch (NoSuchAlgorithmException e) {
- throw new Error("java must always support SHA-1 message digest format", e);
- }
- return str.hashCode();
- }
-
- @Override
- public String toString() {
- return name;
- }
-
- /**
- * An instance of a serialization of this function, using a particular serialization context (by {@link
- * ExpressionFunction#expand})
- */
- public class Instance {
-
- private final String name;
- private final String expressionString;
-
- public Instance(String name, String expressionString) {
- this.name = name;
- this.expressionString = expressionString;
- }
-
- public String getName() {
- return name;
- }
-
- public String getExpressionString() {
- return expressionString;
- }
-
- }
-}
+package com.yahoo.searchlib.rankingexpression; + +import com.google.common.collect.ImmutableList; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; +import com.yahoo.text.Utf8; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; + +/** + * <p>A function defined by a ranking expression</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author bratseth + */ +public class ExpressionFunction { + + private final String name; + private final ImmutableList<String> arguments; + private final RankingExpression body; + + /** + * <p>Constructs a new function</p> + * + * @param name the name of this function + * @param arguments its argument names + * @param body the ranking expression that defines this function + */ + public ExpressionFunction(String name, List<String> arguments, RankingExpression body) { + this.name = name; + this.arguments = arguments==null ? ImmutableList.of() : ImmutableList.copyOf(arguments); + this.body = body; + } + + public String getName() { return name; } + + /** Returns an immutable list of the arguments of this */ + public List<String> arguments() { return arguments; } + + public RankingExpression getBody() { return body; } + + /** + * <p>Create and return an instance of this function based on the given + * arguments. If function calls are nested, this call might produce + * additional scripts.</p> + * + * @param context the context used to expand this + * @param arguments the arguments to instantiate on. + * @param path the expansion path leading to this. + * @return the script function instance created. + */ + public Instance expand(SerializationContext context, List<ExpressionNode> arguments, Deque<String> path) { + Map<String, String> argumentBindings = new HashMap<>(); + for (int i = 0; i < this.arguments.size() && i < arguments.size(); ++i) { + argumentBindings.put(this.arguments.get(i), arguments.get(i).toString(context, path, null)); + } + return new Instance(toSymbol(argumentBindings), body.getRoot().toString(context.createBinding(argumentBindings), path, null)); + } + + /** + * Returns a symbolic string that represents this function with a given + * list of arguments. The arguments are mangled by hashing the string + * representation of the argument expressions, so we might need to revisit + * this if we start seeing collisions. + * + * @param argumentBindings the bound arguments to include in the symbolic name. + * @return the symbolic name for an instance of this function + */ + private String toSymbol(Map<String, String> argumentBindings) { + if (argumentBindings.isEmpty()) return name; + + StringBuilder ret = new StringBuilder(); + ret.append(name).append("@"); + for (Map.Entry<String,String> argumentBinding : argumentBindings.entrySet()) { + ret.append(Long.toHexString(symbolCode(argumentBinding.getKey() + "=" + argumentBinding.getValue()))); + ret.append("."); + } + if (ret.toString().endsWith(".")) + ret.setLength(ret.length()-1); + return ret.toString(); + } + + + /** + * <p>Returns a more unique hash code than what Java's own {@link + * String#hashCode()} method would produce.</p> + * + * @param str The string to hash. + * @return A 64 bit long hash code. + */ + private static long symbolCode(String str) { + try { + MessageDigest md = java.security.MessageDigest.getInstance("SHA-1"); + byte[] buf = md.digest(Utf8.toBytes(str)); + if (buf.length >= 8) { + long ret = 0; + for (int i = 0; i < 8; ++i) { + ret = (ret << 8) + (buf[i] & 0xff); + } + return ret; + } + } catch (NoSuchAlgorithmException e) { + throw new Error("java must always support SHA-1 message digest format", e); + } + return str.hashCode(); + } + + @Override + public String toString() { + return name; + } + + /** + * An instance of a serialization of this function, using a particular serialization context (by {@link + * ExpressionFunction#expand}) + */ + public class Instance { + + private final String name; + private final String expressionString; + + public Instance(String name, String expressionString) { + this.name = name; + this.expressionString = expressionString; + } + + public String getName() { + return name; + } + + public String getExpressionString() { + return expressionString; + } + + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/FeatureList.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/FeatureList.java index 527a908da73..c18a8c0e838 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/FeatureList.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/FeatureList.java @@ -1,140 +1,140 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.rankingexpression;
-
-import com.google.common.annotations.Beta;
-import com.yahoo.searchlib.rankingexpression.parser.ParseException;
-import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParser;
-import com.yahoo.searchlib.rankingexpression.parser.TokenMgrError;
-import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Encapsulates the production rule 'featureList()' int the RankingExpressionParser.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-@Beta
-public class FeatureList implements Iterable<ReferenceNode> {
-
- private final List<ReferenceNode> features = new ArrayList<>();
-
- /**
- * Creates a new feature list by consuming from a reader object.
- *
- * @param reader The reader that contains the string to parse.
- * @throws ParseException Thrown if the string could not be parsed.
- */
- public FeatureList(Reader reader) throws ParseException {
- features.addAll(parse(reader));
- }
-
- /**
- * Creates a new feature list by parsing a string.
- *
- * @param list The string to parse.
- * @throws ParseException Thrown if the string could not be parsed.
- */
- public FeatureList(String list) throws ParseException {
- features.addAll(parse(new StringReader(list)));
- }
-
- /**
- * Creates a new feature list by reading the content of a file.
- *
- * @param file The file whose content to parse.
- * @throws ParseException Thrown if the string could not be parsed.
- * @throws FileNotFoundException Thrown if the file specified could not be found.
- */
- public FeatureList(File file) throws ParseException, FileNotFoundException {
- features.addAll(parse(new FileReader(file)));
- }
-
- /**
- * Parses the content of a reader object as a list of feature nodes.
- *
- * @param reader A reader object that contains an feature list.
- * @return A list of those features named in the string.
- * @throws ParseException if the string could not be parsed.
- */
- private static List<ReferenceNode> parse(Reader reader) throws ParseException {
- List<ReferenceNode> lst;
- try {
- lst = new RankingExpressionParser(reader).featureList();
- }
- catch (TokenMgrError e) {
- ParseException t = new ParseException();
- throw (ParseException)t.initCause(e);
- }
- List<ReferenceNode> ret = new ArrayList<ReferenceNode>(lst.size());
- for (Object obj : lst) {
- if (!(obj instanceof ReferenceNode)) {
- throw new IllegalStateException("Feature list contains a " + obj.getClass().getName() + ".");
- }
- ret.add((ReferenceNode)obj);
- }
- return ret;
- }
-
- /**
- * Returns the number of features in this list.
- *
- * @return The size.
- */
- public int size() {
- return features.size();
- }
-
- /**
- * Returns the feature at the given index.
- *
- * @param i The index of the feature to return.
- * @return The featuer at the given index.
- */
- public ReferenceNode get(int i) {
- return features.get(i);
- }
-
- @Override
- public int hashCode() {
- int ret = 0;
- for (ReferenceNode node : features) {
- ret += node.hashCode() * 17;
- }
- return ret;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof FeatureList)) {
- return false;
- }
- FeatureList lst = (FeatureList)obj;
- if (features.size() != lst.features.size()) {
- return false;
- }
- for (int i = 0; i < features.size(); ++i) {
- if (!features.get(i).equals(lst.features.get(i))) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public String toString() {
- StringBuilder ret = new StringBuilder();
- for (ReferenceNode node : this) {
- ret.append(node).append(" ");
- }
- return ret.toString();
- }
-
- @Override
- public Iterator<ReferenceNode> iterator() {
- return features.iterator();
- }
-}
+package com.yahoo.searchlib.rankingexpression; + +import com.google.common.annotations.Beta; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParser; +import com.yahoo.searchlib.rankingexpression.parser.TokenMgrError; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; + +import java.io.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Encapsulates the production rule 'featureList()' int the RankingExpressionParser. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@Beta +public class FeatureList implements Iterable<ReferenceNode> { + + private final List<ReferenceNode> features = new ArrayList<>(); + + /** + * Creates a new feature list by consuming from a reader object. + * + * @param reader The reader that contains the string to parse. + * @throws ParseException Thrown if the string could not be parsed. + */ + public FeatureList(Reader reader) throws ParseException { + features.addAll(parse(reader)); + } + + /** + * Creates a new feature list by parsing a string. + * + * @param list The string to parse. + * @throws ParseException Thrown if the string could not be parsed. + */ + public FeatureList(String list) throws ParseException { + features.addAll(parse(new StringReader(list))); + } + + /** + * Creates a new feature list by reading the content of a file. + * + * @param file The file whose content to parse. + * @throws ParseException Thrown if the string could not be parsed. + * @throws FileNotFoundException Thrown if the file specified could not be found. + */ + public FeatureList(File file) throws ParseException, FileNotFoundException { + features.addAll(parse(new FileReader(file))); + } + + /** + * Parses the content of a reader object as a list of feature nodes. + * + * @param reader A reader object that contains an feature list. + * @return A list of those features named in the string. + * @throws ParseException if the string could not be parsed. + */ + private static List<ReferenceNode> parse(Reader reader) throws ParseException { + List<ReferenceNode> lst; + try { + lst = new RankingExpressionParser(reader).featureList(); + } + catch (TokenMgrError e) { + ParseException t = new ParseException(); + throw (ParseException)t.initCause(e); + } + List<ReferenceNode> ret = new ArrayList<ReferenceNode>(lst.size()); + for (Object obj : lst) { + if (!(obj instanceof ReferenceNode)) { + throw new IllegalStateException("Feature list contains a " + obj.getClass().getName() + "."); + } + ret.add((ReferenceNode)obj); + } + return ret; + } + + /** + * Returns the number of features in this list. + * + * @return The size. + */ + public int size() { + return features.size(); + } + + /** + * Returns the feature at the given index. + * + * @param i The index of the feature to return. + * @return The featuer at the given index. + */ + public ReferenceNode get(int i) { + return features.get(i); + } + + @Override + public int hashCode() { + int ret = 0; + for (ReferenceNode node : features) { + ret += node.hashCode() * 17; + } + return ret; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof FeatureList)) { + return false; + } + FeatureList lst = (FeatureList)obj; + if (features.size() != lst.features.size()) { + return false; + } + for (int i = 0; i < features.size(); ++i) { + if (!features.get(i).equals(lst.features.get(i))) { + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + for (ReferenceNode node : this) { + ret.append(node).append(" "); + } + return ret.toString(); + } + + @Override + public Iterator<ReferenceNode> iterator() { + return features.iterator(); + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java index 49604f14ca2..fdb71bca2f5 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java @@ -1,57 +1,57 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchlib.rankingexpression.rule;
-
-import com.yahoo.searchlib.rankingexpression.evaluation.Context;
-import com.yahoo.searchlib.rankingexpression.evaluation.Value;
-
-import java.util.Collections;
-import java.util.Deque;
-import java.util.List;
-
-/**
- * This class represents another expression enclosed in braces.
- *
- * @author Simon Thoresen
- */
-public final class EmbracedNode extends CompositeNode {
-
- // The node to embrace.
- private final ExpressionNode value;
-
- /**
- * Creates a new expression node that embraces another.
- *
- * @param value The node to embrace.
- */
- public EmbracedNode(ExpressionNode value) {
- this.value=value;
- }
-
- /** Returns the node enclosed by this */
- public ExpressionNode getValue() { return value; }
-
- @Override
- public List<ExpressionNode> children() {
- return Collections.singletonList(value);
- }
-
- @Override
- public String toString(SerializationContext context, Deque<String> path, CompositeNode parent) {
- String expression = value.toString(context, path, this);
- if (value instanceof ReferenceNode) return expression;
- return "(" + expression + ")";
- }
-
- @Override
- public Value evaluate(Context context) {
- return value.evaluate(context);
- }
-
- @Override
- public CompositeNode setChildren(List<ExpressionNode> newChildren) {
- if (newChildren.size() != 1)
- throw new IllegalArgumentException("Expected 1 child but got " + newChildren.size());
- return new EmbracedNode(newChildren.get(0));
- }
-
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchlib.rankingexpression.rule; + +import com.yahoo.searchlib.rankingexpression.evaluation.Context; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; + +import java.util.Collections; +import java.util.Deque; +import java.util.List; + +/** + * This class represents another expression enclosed in braces. + * + * @author Simon Thoresen + */ +public final class EmbracedNode extends CompositeNode { + + // The node to embrace. + private final ExpressionNode value; + + /** + * Creates a new expression node that embraces another. + * + * @param value The node to embrace. + */ + public EmbracedNode(ExpressionNode value) { + this.value=value; + } + + /** Returns the node enclosed by this */ + public ExpressionNode getValue() { return value; } + + @Override + public List<ExpressionNode> children() { + return Collections.singletonList(value); + } + + @Override + public String toString(SerializationContext context, Deque<String> path, CompositeNode parent) { + String expression = value.toString(context, path, this); + if (value instanceof ReferenceNode) return expression; + return "(" + expression + ")"; + } + + @Override + public Value evaluate(Context context) { + return value.evaluate(context); + } + + @Override + public CompositeNode setChildren(List<ExpressionNode> newChildren) { + if (newChildren.size() != 1) + throw new IllegalArgumentException("Expected 1 child but got " + newChildren.size()); + return new EmbracedNode(newChildren.get(0)); + } + +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java index 234825eaa3d..1b74326b168 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TruthOperator.java @@ -47,4 +47,4 @@ public enum TruthOperator implements Serializable { return x==y; } -}
\ No newline at end of file +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/TreeNetConverter.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/TreeNetConverter.java index c147c3a33b8..628cfc10a48 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/TreeNetConverter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/TreeNetConverter.java @@ -1,35 +1,35 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet;
-
-import com.yahoo.searchlib.treenet.parser.TreeNetParser;
-
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class TreeNetConverter {
-
- /**
- * Implements an application main function so that the converter can be used as a command-line tool.
- *
- * @param args List of arguments.
- */
- public static void main(String[] args) {
- if (args.length != 1) {
- System.err.println("Usage: TreeNetConverter <filename>");
- System.exit(1);
- }
- try {
- TreeNetParser parser = new TreeNetParser(new FileReader(args[0]));
- System.out.println(parser.treeNet().toRankingExpression());
- } catch (FileNotFoundException e) {
- System.err.println("Could not find file '" + args[0] + "'.");
- System.exit(1);
- } catch (Exception e) {
- System.err.println("An error occured while parsing the content of file '" + args[0] + "': " + e);
- System.exit(1);
- }
- }
-}
+package com.yahoo.searchlib.treenet; + +import com.yahoo.searchlib.treenet.parser.TreeNetParser; + +import java.io.FileNotFoundException; +import java.io.FileReader; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TreeNetConverter { + + /** + * Implements an application main function so that the converter can be used as a command-line tool. + * + * @param args List of arguments. + */ + public static void main(String[] args) { + if (args.length != 1) { + System.err.println("Usage: TreeNetConverter <filename>"); + System.exit(1); + } + try { + TreeNetParser parser = new TreeNetParser(new FileReader(args[0])); + System.out.println(parser.treeNet().toRankingExpression()); + } catch (FileNotFoundException e) { + System.err.println("Could not find file '" + args[0] + "'."); + System.exit(1); + } catch (Exception e) { + System.err.println("An error occured while parsing the content of file '" + args[0] + "': " + e); + System.exit(1); + } + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/ComparisonCondition.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/ComparisonCondition.java index 1855a8a5674..6c40406977b 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/ComparisonCondition.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/ComparisonCondition.java @@ -1,39 +1,39 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet.rule;
-
-import com.yahoo.java7compat.Util;
-
-/**
- * Represents a condition which comparing two values
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class ComparisonCondition extends Condition {
-
- private final double rhs;
-
- /**
- * Constructs a new instance of this class.
- *
- * @param lhs The name of the feature to compare to a constant.
- * @param rhs The constant to compare the feature with.
- * @param ift The label to jump to if left < right.
- * @param iff The label to jump to if left >= right;
- */
- public ComparisonCondition(String lhs, double rhs, String ift, String iff) {
- super(lhs, ift, iff);
- this.rhs = rhs;
- }
-
- /**
- * Returns the constant to compare the feature with.
- *
- * @return The constant.
- */
- public double getConstant() { return rhs; }
-
- @Override
- public String conditionToRankingExpression() {
- return "< " + Util.toJava7String(rhs);
- }
-}
+package com.yahoo.searchlib.treenet.rule; + +import com.yahoo.java7compat.Util; + +/** + * Represents a condition which comparing two values + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ComparisonCondition extends Condition { + + private final double rhs; + + /** + * Constructs a new instance of this class. + * + * @param lhs The name of the feature to compare to a constant. + * @param rhs The constant to compare the feature with. + * @param ift The label to jump to if left < right. + * @param iff The label to jump to if left >= right; + */ + public ComparisonCondition(String lhs, double rhs, String ift, String iff) { + super(lhs, ift, iff); + this.rhs = rhs; + } + + /** + * Returns the constant to compare the feature with. + * + * @return The constant. + */ + public double getConstant() { return rhs; } + + @Override + public String conditionToRankingExpression() { + return "< " + Util.toJava7String(rhs); + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Response.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Response.java index 347dd84f419..9838cafcab3 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Response.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Response.java @@ -1,45 +1,45 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet.rule;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class Response extends TreeNode {
-
- // The id of the next tree to run after this.
- private final Double value;
-
- // The value of this response.
- private final String next;
-
- /**
- * Constructs a new response.
- *
- * @param next The id of the next tree to run after this.
- * @param value The value of this response.
- */
- public Response(Double value, String next) {
- super();
- this.value = value;
- this.next = next;
- }
-
- /**
- * Returns the value of this response.
- */
- public Double getValue() {
- return value;
- }
-
- /**
- * Returns the id of the next tree to run after this.
- */
- public String getNext() {
- return next;
- }
-
- @Override
- public String toRankingExpression() {
- return value.toString();
- }
-}
+package com.yahoo.searchlib.treenet.rule; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Response extends TreeNode { + + // The id of the next tree to run after this. + private final Double value; + + // The value of this response. + private final String next; + + /** + * Constructs a new response. + * + * @param next The id of the next tree to run after this. + * @param value The value of this response. + */ + public Response(Double value, String next) { + super(); + this.value = value; + this.next = next; + } + + /** + * Returns the value of this response. + */ + public Double getValue() { + return value; + } + + /** + * Returns the id of the next tree to run after this. + */ + public String getNext() { + return next; + } + + @Override + public String toRankingExpression() { + return value.toString(); + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/SetMembershipCondition.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/SetMembershipCondition.java index bb47947557c..a992d3b8032 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/SetMembershipCondition.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/SetMembershipCondition.java @@ -1,57 +1,57 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchlib.treenet.rule;
-
-import com.yahoo.java7compat.Util;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Represents a set membership test on the form <code>feature IN (integer1, integer2 ...)</code>
- *
- * @author bratseth
- * @since 5.1.21
- */
-public class SetMembershipCondition extends Condition {
-
- private final List<Object> setValues;
-
- /**
- * Constructs a new instance of this class.
- *
- * @param testValue the name of the feature to test
- * @param setValues the set of values to compare to
- * @param trueLabel the label to jump to if the value is in the set
- * @param falseLabel the label to jumt to if the value is not in the set
- */
- public SetMembershipCondition(String testValue, List<Object> setValues, String trueLabel, String falseLabel) {
- super(testValue, trueLabel, falseLabel);
- this.setValues = Collections.unmodifiableList(new ArrayList<>(setValues));
- }
-
- /** Returns the unmodifiable set of values to check */
- public List<Object> getSetValues() { return setValues; }
-
- @Override
- protected String conditionToRankingExpression() {
- StringBuilder b = new StringBuilder("in [");
- for (Iterator<Object> i = setValues.iterator(); i.hasNext(); ) {
- Object value = i.next();
- if (value instanceof String)
- b.append("\"").append(value).append("\"");
- else if (value instanceof Integer)
- b.append(value);
- else
- throw new RuntimeException("Excepted a string or integer in a set membership test, not a " +
- value.getClass() + ": " + value);
-
- if (i.hasNext())
- b.append(",");
- }
- b.append("]");
- return b.toString();
- }
-
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchlib.treenet.rule; + +import com.yahoo.java7compat.Util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * Represents a set membership test on the form <code>feature IN (integer1, integer2 ...)</code> + * + * @author bratseth + * @since 5.1.21 + */ +public class SetMembershipCondition extends Condition { + + private final List<Object> setValues; + + /** + * Constructs a new instance of this class. + * + * @param testValue the name of the feature to test + * @param setValues the set of values to compare to + * @param trueLabel the label to jump to if the value is in the set + * @param falseLabel the label to jumt to if the value is not in the set + */ + public SetMembershipCondition(String testValue, List<Object> setValues, String trueLabel, String falseLabel) { + super(testValue, trueLabel, falseLabel); + this.setValues = Collections.unmodifiableList(new ArrayList<>(setValues)); + } + + /** Returns the unmodifiable set of values to check */ + public List<Object> getSetValues() { return setValues; } + + @Override + protected String conditionToRankingExpression() { + StringBuilder b = new StringBuilder("in ["); + for (Iterator<Object> i = setValues.iterator(); i.hasNext(); ) { + Object value = i.next(); + if (value instanceof String) + b.append("\"").append(value).append("\""); + else if (value instanceof Integer) + b.append(value); + else + throw new RuntimeException("Excepted a string or integer in a set membership test, not a " + + value.getClass() + ": " + value); + + if (i.hasNext()) + b.append(","); + } + b.append("]"); + return b.toString(); + } + +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Tree.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Tree.java index 2a7191baeba..6db319ebc6c 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Tree.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/Tree.java @@ -1,110 +1,110 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet.rule;
-
-import java.util.Map;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class Tree {
-
- private final String name;
-
- // The parent tree net of this.
- private TreeNet parent;
-
- // Returns the id of the next tree to run after this.
- private String next;
-
- // The initial response value of this tree, may be null.
- private final Double value;
-
- // The id of the first condition or response to run in this tree.
- private final String begin;
-
- // All named nodes of this tree.
- private final Map<String, TreeNode> nodes;
-
- /**
- * Constructs a new tree.
- *
- * @param name The name of this tree, used for error outputs.
- * @param value The initial response value of this tree, may be null.
- * @param begin The id of the first condition or response to run in this tree.
- * @param nodes All named nodes of this tree.
- */
- public Tree(String name, Double value, String begin, Map<String, TreeNode> nodes) {
- this.name = name;
- this.value = value;
- this.begin = begin;
- this.nodes = nodes;
-
- this.next = null;
- for (TreeNode node : this.nodes.values()) {
- node.setParent(this);
- if (node instanceof Response) {
- String next = ((Response)node).getNext();
- if (this.next == null) {
- this.next = next;
- } else if (!this.next.equals(next)) {
- throw new IllegalStateException("Not all child nodes of tree '" + name + "' agree on the next " +
- "tree to run. Initial name was '" + this.next + "', conflicting " +
- "name is '" + next + "'.");
- }
- }
- }
- }
-
- public String getName() { return name; }
-
- /**
- * Returns the parent tree net of this.
- */
- public TreeNet getParent() { return parent; }
-
- /**
- * Sets the parent tree net of this.
- *
- * @param parent The parent tree net.
- * @return This, to allow chaining.
- */
- public Tree setParent(TreeNet parent) {
- this.parent = parent;
- return this;
- }
-
- /**
- * Returns the id of the next tree to run after this.
- */
- public String getNext() {
- return next;
- }
-
- /**
- * Returns the initial response value of this tree, may be null.
- */
- public Double getValue() {
- return value;
- }
-
- /**
- * Returns the id of the first condition or response to run in this tree.
- */
- public String getBegin() {
- return begin;
- }
-
- /**
- * Returns all named nodes of this tree.
- */
- public Map<String, TreeNode> getNodes() {
- return nodes;
- }
-
- /**
- * Returns a ranking expression equivalent of this tree.
- */
- public String toRankingExpression() {
- return nodes.get(begin).toRankingExpression();
- }
-}
+package com.yahoo.searchlib.treenet.rule; + +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Tree { + + private final String name; + + // The parent tree net of this. + private TreeNet parent; + + // Returns the id of the next tree to run after this. + private String next; + + // The initial response value of this tree, may be null. + private final Double value; + + // The id of the first condition or response to run in this tree. + private final String begin; + + // All named nodes of this tree. + private final Map<String, TreeNode> nodes; + + /** + * Constructs a new tree. + * + * @param name The name of this tree, used for error outputs. + * @param value The initial response value of this tree, may be null. + * @param begin The id of the first condition or response to run in this tree. + * @param nodes All named nodes of this tree. + */ + public Tree(String name, Double value, String begin, Map<String, TreeNode> nodes) { + this.name = name; + this.value = value; + this.begin = begin; + this.nodes = nodes; + + this.next = null; + for (TreeNode node : this.nodes.values()) { + node.setParent(this); + if (node instanceof Response) { + String next = ((Response)node).getNext(); + if (this.next == null) { + this.next = next; + } else if (!this.next.equals(next)) { + throw new IllegalStateException("Not all child nodes of tree '" + name + "' agree on the next " + + "tree to run. Initial name was '" + this.next + "', conflicting " + + "name is '" + next + "'."); + } + } + } + } + + public String getName() { return name; } + + /** + * Returns the parent tree net of this. + */ + public TreeNet getParent() { return parent; } + + /** + * Sets the parent tree net of this. + * + * @param parent The parent tree net. + * @return This, to allow chaining. + */ + public Tree setParent(TreeNet parent) { + this.parent = parent; + return this; + } + + /** + * Returns the id of the next tree to run after this. + */ + public String getNext() { + return next; + } + + /** + * Returns the initial response value of this tree, may be null. + */ + public Double getValue() { + return value; + } + + /** + * Returns the id of the first condition or response to run in this tree. + */ + public String getBegin() { + return begin; + } + + /** + * Returns all named nodes of this tree. + */ + public Map<String, TreeNode> getNodes() { + return nodes; + } + + /** + * Returns a ranking expression equivalent of this tree. + */ + public String toRankingExpression() { + return nodes.get(begin).toRankingExpression(); + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNet.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNet.java index 1db13b6c12e..47a2afad4d2 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNet.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNet.java @@ -1,63 +1,63 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet.rule;
-
-import java.util.Map;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class TreeNet {
-
- // The id of the first tree to run in this net.
- private String begin;
-
- // All named trees of this net.
- private final Map<String, Tree> trees;
-
- /**
- * Constructs a new tree net.
- *
- * @param begin The id of the first tree to run in this net.
- * @param trees All named trees of this net.
- */
- public TreeNet(String begin, Map<String, Tree> trees) {
- this.begin = begin;
- this.trees = trees;
- for (Tree tree : this.trees.values()) {
- tree.setParent(this);
- }
- }
-
- /**
- * Returns the id of the first tree to run in this net.
- */
- public String getBegin() {
- return begin;
- }
-
- /**
- * Returns all named trees of this net.
- */
- public Map<String, Tree> getTrees() {
- return trees;
- }
-
- /**
- * Returns a ranking expression equivalent of this net.
- */
- public String toRankingExpression() {
- StringBuilder ret = new StringBuilder();
- String next = begin;
- while (next != null) {
- Tree tree = trees.get(next);
- if (tree.getBegin() != null) {
- if (ret.length() > 0) {
- ret.append(" + \n");
- }
- ret.append(tree.toRankingExpression());
- }
- next = tree.getNext();
- }
- return ret.toString();
- }
-}
+package com.yahoo.searchlib.treenet.rule; + +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TreeNet { + + // The id of the first tree to run in this net. + private String begin; + + // All named trees of this net. + private final Map<String, Tree> trees; + + /** + * Constructs a new tree net. + * + * @param begin The id of the first tree to run in this net. + * @param trees All named trees of this net. + */ + public TreeNet(String begin, Map<String, Tree> trees) { + this.begin = begin; + this.trees = trees; + for (Tree tree : this.trees.values()) { + tree.setParent(this); + } + } + + /** + * Returns the id of the first tree to run in this net. + */ + public String getBegin() { + return begin; + } + + /** + * Returns all named trees of this net. + */ + public Map<String, Tree> getTrees() { + return trees; + } + + /** + * Returns a ranking expression equivalent of this net. + */ + public String toRankingExpression() { + StringBuilder ret = new StringBuilder(); + String next = begin; + while (next != null) { + Tree tree = trees.get(next); + if (tree.getBegin() != null) { + if (ret.length() > 0) { + ret.append(" + \n"); + } + ret.append(tree.toRankingExpression()); + } + next = tree.getNext(); + } + return ret.toString(); + } +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNode.java b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNode.java index a637adafc73..178d6810193 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/treenet/rule/TreeNode.java @@ -1,34 +1,34 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet.rule;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public abstract class TreeNode {
-
- // The parent tree of this.
- private Tree parent = null;
-
- /**
- * Returns the parent tree of this.
- */
- public Tree getParent() {
- return parent;
- }
-
- /**
- * Sets the parent tree net of this.
- *
- * @param parent The parent tree net.
- * @return This, to allow chaining.
- */
- public TreeNode setParent(Tree parent) {
- this.parent = parent;
- return this;
- }
-
- /**
- * Returns a ranking expression equivalent of this net.
- */
- public abstract String toRankingExpression();
-}
+package com.yahoo.searchlib.treenet.rule; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class TreeNode { + + // The parent tree of this. + private Tree parent = null; + + /** + * Returns the parent tree of this. + */ + public Tree getParent() { + return parent; + } + + /** + * Sets the parent tree net of this. + * + * @param parent The parent tree net. + * @return This, to allow chaining. + */ + public TreeNode setParent(Tree parent) { + this.parent = parent; + return this; + } + + /** + * Returns a ranking expression equivalent of this net. + */ + public abstract String toRankingExpression(); +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/BiasEstimatorTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/BiasEstimatorTest.java index 307214d8c1c..c9e510736c9 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/BiasEstimatorTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/BiasEstimatorTest.java @@ -67,4 +67,4 @@ public class BiasEstimatorTest { private static void assertEstimateEquals(double expected, double rawEstimate, BiasEstimator biasEstimator) { assertEquals(expected, biasEstimator.estimateBias(rawEstimate), 0.00000001); } -}
\ No newline at end of file +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/HyperLogLogEstimatorTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/HyperLogLogEstimatorTest.java index 1ba4a71d102..28ebc385194 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/HyperLogLogEstimatorTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/HyperLogLogEstimatorTest.java @@ -86,4 +86,4 @@ public class HyperLogLogEstimatorTest { byte[] bytes = ByteBuffer.allocate(4).putInt(value).array(); return hashGenerator.hash(bytes, 0, 4, seed); } -}
\ No newline at end of file +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java index 3b0a584f37b..b1f9d2cd96c 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/NormalSketchTest.java @@ -118,4 +118,4 @@ public class NormalSketchTest { } } -}
\ No newline at end of file +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchMergerTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchMergerTest.java index 07488d21fd3..e435e76caa7 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchMergerTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SketchMergerTest.java @@ -66,4 +66,4 @@ public class SketchMergerTest { assertTrue("Should return the NormalSketch instance given by the arguments.", result == s2); SketchUtils.assertSketchContains(result, 1, 2); } -}
\ No newline at end of file +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SparseSketchTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SparseSketchTest.java index 4be0f89514d..889050959e8 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SparseSketchTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/hll/SparseSketchTest.java @@ -59,4 +59,4 @@ public class SparseSketchTest { assertEquals(s1.data(), s2.data()); } -}
\ No newline at end of file +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/gbdt/TreeNodeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/gbdt/TreeNodeTestCase.java index 572bd2d8c11..2e39104f136 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/gbdt/TreeNodeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/gbdt/TreeNodeTestCase.java @@ -54,4 +54,4 @@ public class TreeNodeTestCase { assertEquals("Unknown", e.getMessage()); } } -}
\ No newline at end of file +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java index 7399088ac1c..dfa726ecb8f 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/FeatureListTestCase.java @@ -1,77 +1,77 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.rankingexpression;
-
-import com.yahoo.searchlib.rankingexpression.parser.ParseException;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class FeatureListTestCase {
-
- @Test
- public void requireThatFeatureListFromStringWorks() throws ParseException {
- assertFromString("attribute(foo).out",
- Arrays.asList("attribute(foo).out"));
- assertFromString("attribute(foo).out attribute ( bar ) . out",
- Arrays.asList("attribute(foo).out", "attribute(bar).out"));
- assertFromString("foo\n bar\n \t \t \n baz \n",
- Arrays.asList("foo", "bar", "baz"));
- assertFromString("attribute attribute(foo) attribute(foo).out attribute(bar).out.out",
- Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out"));
- }
-
- @Test
- public void requireThatFeatureListFromReaderWorks() throws ParseException {
- assertFromReader(new StringReader("attribute(foo).out"),
- Arrays.asList("attribute(foo).out"));
- assertFromReader(new StringReader("attribute(foo).out attribute ( bar ) . out"),
- Arrays.asList("attribute(foo).out", "attribute(bar).out"));
- assertFromReader(new StringReader("foo\n bar\n \t \t \n baz \n"),
- Arrays.asList("foo", "bar", "baz"));
- assertFromReader(new StringReader("attribute attribute(foo) attribute(foo).out attribute(bar).out.out"),
- Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out"));
- }
-
- @Test
- public void requireThatFeatureListFromFileWorks() throws ParseException, FileNotFoundException {
- assertFromFile(new File("src/test/files/features01.expression"),
- Arrays.asList("attribute(foo).out"));
- assertFromFile(new File("src/test/files/features02.expression"),
- Arrays.asList("attribute(foo).out", "attribute(bar).out"));
- assertFromFile(new File("src/test/files/features03.expression"),
- Arrays.asList("foo", "bar", "baz"));
- assertFromFile(new File("src/test/files/features04.expression"),
- Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out"));
- }
-
- public void assertFromString(String input, List<String> expected) throws ParseException {
- assertFeatureList(new FeatureList(input), expected);
- }
-
- public void assertFromReader(Reader input, List<String> expected) throws ParseException {
- assertFeatureList(new FeatureList(input), expected);
- }
-
- public void assertFromFile(File input, List<String> expected) throws ParseException, FileNotFoundException {
- assertFeatureList(new FeatureList(input), expected);
- }
-
- public void assertFeatureList(FeatureList features, List<String> expected) throws ParseException {
- assertEquals(expected.size(), features.size());
- for (int i = 0; i < features.size(); ++i) {
- assertTrue(features.get(i) != null);
- assertEquals(expected.get(i), features.get(i).toString());
- }
- }
-}
+package com.yahoo.searchlib.rankingexpression; + +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.Reader; +import java.io.StringReader; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FeatureListTestCase { + + @Test + public void requireThatFeatureListFromStringWorks() throws ParseException { + assertFromString("attribute(foo).out", + Arrays.asList("attribute(foo).out")); + assertFromString("attribute(foo).out attribute ( bar ) . out", + Arrays.asList("attribute(foo).out", "attribute(bar).out")); + assertFromString("foo\n bar\n \t \t \n baz \n", + Arrays.asList("foo", "bar", "baz")); + assertFromString("attribute attribute(foo) attribute(foo).out attribute(bar).out.out", + Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); + } + + @Test + public void requireThatFeatureListFromReaderWorks() throws ParseException { + assertFromReader(new StringReader("attribute(foo).out"), + Arrays.asList("attribute(foo).out")); + assertFromReader(new StringReader("attribute(foo).out attribute ( bar ) . out"), + Arrays.asList("attribute(foo).out", "attribute(bar).out")); + assertFromReader(new StringReader("foo\n bar\n \t \t \n baz \n"), + Arrays.asList("foo", "bar", "baz")); + assertFromReader(new StringReader("attribute attribute(foo) attribute(foo).out attribute(bar).out.out"), + Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); + } + + @Test + public void requireThatFeatureListFromFileWorks() throws ParseException, FileNotFoundException { + assertFromFile(new File("src/test/files/features01.expression"), + Arrays.asList("attribute(foo).out")); + assertFromFile(new File("src/test/files/features02.expression"), + Arrays.asList("attribute(foo).out", "attribute(bar).out")); + assertFromFile(new File("src/test/files/features03.expression"), + Arrays.asList("foo", "bar", "baz")); + assertFromFile(new File("src/test/files/features04.expression"), + Arrays.asList("attribute", "attribute(foo)", "attribute(foo).out", "attribute(bar).out.out")); + } + + public void assertFromString(String input, List<String> expected) throws ParseException { + assertFeatureList(new FeatureList(input), expected); + } + + public void assertFromReader(Reader input, List<String> expected) throws ParseException { + assertFeatureList(new FeatureList(input), expected); + } + + public void assertFromFile(File input, List<String> expected) throws ParseException, FileNotFoundException { + assertFeatureList(new FeatureList(input), expected); + } + + public void assertFeatureList(FeatureList features, List<String> expected) throws ParseException { + assertEquals(expected.size(), features.size()); + for (int i = 0; i < features.size(); ++i) { + assertTrue(features.get(i) != null); + assertEquals(expected.get(i), features.get(i).toString()); + } + } +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/Benchmark.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/Benchmark.java index 7690efb1112..b2633f13fe6 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/Benchmark.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/Benchmark.java @@ -1,144 +1,144 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.rankingexpression.evaluation;
-
-import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.GBDTForestOptimizer;
-import com.yahoo.searchlib.rankingexpression.parser.ParseException;
-import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
-import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
-import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public final class Benchmark {
-
- public static void main(String[] args) {
- if (args.length < 1) {
- System.err.println("Usage: Benchmark <filename> [<iterations>]");
- System.exit(1);
- }
- int numRuns = 1000;
- if (args.length == 2) {
- numRuns = Integer.valueOf(args[1]);
- }
- List<Result> res = new ArrayList<Result>();
- try {
- BufferedReader in = new BufferedReader(new FileReader(args[0]));
- StringBuilder str = new StringBuilder();
- String line;
- while ((line = in.readLine()) != null) {
- str.append(line);
- }
- String exp = str.toString();
- res.add(evaluateTree(exp, numRuns));
- res.add(evaluateTreeOptimized(exp, numRuns));
- res.add(evaluateForestOptimized(exp, numRuns));
- } catch (IOException e) {
- System.out.println("An error occured while reading the content of file '" + args[0] + "': " + e);
- System.exit(1);
- } catch (ParseException e) {
- System.out.println("An error occured while parsing the content of file '" + args[0] + "': " + e);
- System.exit(1);
- }
- for (Result lhs : res) {
- for (Result rhs : res) {
- if (lhs.res < rhs.res - 1e-6 || lhs.res > rhs.res + 1e-6) {
- System.err.println("Evaluation of '" + lhs.name + "' and '" + rhs.name + "' disagree on result; " +
- "expected " + lhs.res + ", got " + rhs.res + ".");
- System.exit(1);
- }
- }
- System.out.format("%1$-16s : %2$8.04f ms (%3$-6.04f)\n",
- lhs.name, lhs.millis, res.get(0).millis / lhs.millis);
- }
- }
-
- private static Result evaluateTree(String str, int numRuns) throws ParseException {
- Result ret = new Result();
- ret.name = "Unoptimized";
-
- RankingExpression exp = new RankingExpression(str);
- List<String> vars = new LinkedList<String>();
- getFeatures(exp.getRoot(), vars);
-
- benchmark(exp, vars, new MapContext(), numRuns, ret);
- return ret;
- }
-
- private static Result evaluateTreeOptimized(String str, int numRuns) throws ParseException {
- Result ret = new Result();
- ret.name = "Optimized tree";
-
- RankingExpression exp = new RankingExpression(str);
- List<String> vars = new LinkedList<String>();
- getFeatures(exp.getRoot(), vars);
-
- ArrayContext ctx = new ArrayContext(exp);
- ExpressionOptimizer optimizer = new ExpressionOptimizer();
- optimizer.getOptimizer(GBDTForestOptimizer.class).setEnabled(false);
- optimizer.optimize(exp, ctx);
-
- benchmark(exp, vars, ctx, numRuns, ret);
- return ret;
- }
-
- private static Result evaluateForestOptimized(String str, int numRuns) throws ParseException {
- Result ret = new Result();
- ret.name = "Optimized forest";
-
- RankingExpression exp = new RankingExpression(str);
- List<String> vars = new LinkedList<String>();
- getFeatures(exp.getRoot(), vars);
-
- ArrayContext ctx = new ArrayContext(exp);
- ExpressionOptimizer optimizer = new ExpressionOptimizer();
- optimizer.optimize(exp, ctx);
-
- benchmark(exp, vars, ctx, numRuns, ret);
- return ret;
- }
-
- private static void benchmark(RankingExpression exp, List<String> vars, Context ctx, int numRuns, Result out) {
- for (int i = 0, len = vars.size(); i < len; ++i) {
- ctx.put(vars.get(i), i / (double)len);
- }
- for (int i = 0; i < numRuns; ++i) {
- out.res = exp.evaluate(ctx).asDouble();
- }
- long begin = System.nanoTime();
- for (int i = 0; i < numRuns; ++i) {
- out.res = exp.evaluate(ctx).asDouble();
- }
- long end = System.nanoTime();
-
- out.millis = (end - begin) / (1000.0 * 1000.0);
- }
-
- private static void getFeatures(ExpressionNode node, List<String> out) {
- if (node instanceof ReferenceNode) {
- String feature = ((ReferenceNode)node).getName();
- if (!out.contains(feature)) {
- out.add(feature);
- }
- } else if (node instanceof CompositeNode) {
- CompositeNode cNode = (CompositeNode)node;
- for (ExpressionNode child : cNode.children()) {
- getFeatures(child, out);
- }
- }
- }
-
- private static class Result {
- String name = "anonymous";
- double millis = Double.MAX_VALUE;
- double res = 0;
- }
-}
+package com.yahoo.searchlib.rankingexpression.evaluation; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.GBDTForestOptimizer; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public final class Benchmark { + + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Usage: Benchmark <filename> [<iterations>]"); + System.exit(1); + } + int numRuns = 1000; + if (args.length == 2) { + numRuns = Integer.valueOf(args[1]); + } + List<Result> res = new ArrayList<Result>(); + try { + BufferedReader in = new BufferedReader(new FileReader(args[0])); + StringBuilder str = new StringBuilder(); + String line; + while ((line = in.readLine()) != null) { + str.append(line); + } + String exp = str.toString(); + res.add(evaluateTree(exp, numRuns)); + res.add(evaluateTreeOptimized(exp, numRuns)); + res.add(evaluateForestOptimized(exp, numRuns)); + } catch (IOException e) { + System.out.println("An error occured while reading the content of file '" + args[0] + "': " + e); + System.exit(1); + } catch (ParseException e) { + System.out.println("An error occured while parsing the content of file '" + args[0] + "': " + e); + System.exit(1); + } + for (Result lhs : res) { + for (Result rhs : res) { + if (lhs.res < rhs.res - 1e-6 || lhs.res > rhs.res + 1e-6) { + System.err.println("Evaluation of '" + lhs.name + "' and '" + rhs.name + "' disagree on result; " + + "expected " + lhs.res + ", got " + rhs.res + "."); + System.exit(1); + } + } + System.out.format("%1$-16s : %2$8.04f ms (%3$-6.04f)\n", + lhs.name, lhs.millis, res.get(0).millis / lhs.millis); + } + } + + private static Result evaluateTree(String str, int numRuns) throws ParseException { + Result ret = new Result(); + ret.name = "Unoptimized"; + + RankingExpression exp = new RankingExpression(str); + List<String> vars = new LinkedList<String>(); + getFeatures(exp.getRoot(), vars); + + benchmark(exp, vars, new MapContext(), numRuns, ret); + return ret; + } + + private static Result evaluateTreeOptimized(String str, int numRuns) throws ParseException { + Result ret = new Result(); + ret.name = "Optimized tree"; + + RankingExpression exp = new RankingExpression(str); + List<String> vars = new LinkedList<String>(); + getFeatures(exp.getRoot(), vars); + + ArrayContext ctx = new ArrayContext(exp); + ExpressionOptimizer optimizer = new ExpressionOptimizer(); + optimizer.getOptimizer(GBDTForestOptimizer.class).setEnabled(false); + optimizer.optimize(exp, ctx); + + benchmark(exp, vars, ctx, numRuns, ret); + return ret; + } + + private static Result evaluateForestOptimized(String str, int numRuns) throws ParseException { + Result ret = new Result(); + ret.name = "Optimized forest"; + + RankingExpression exp = new RankingExpression(str); + List<String> vars = new LinkedList<String>(); + getFeatures(exp.getRoot(), vars); + + ArrayContext ctx = new ArrayContext(exp); + ExpressionOptimizer optimizer = new ExpressionOptimizer(); + optimizer.optimize(exp, ctx); + + benchmark(exp, vars, ctx, numRuns, ret); + return ret; + } + + private static void benchmark(RankingExpression exp, List<String> vars, Context ctx, int numRuns, Result out) { + for (int i = 0, len = vars.size(); i < len; ++i) { + ctx.put(vars.get(i), i / (double)len); + } + for (int i = 0; i < numRuns; ++i) { + out.res = exp.evaluate(ctx).asDouble(); + } + long begin = System.nanoTime(); + for (int i = 0; i < numRuns; ++i) { + out.res = exp.evaluate(ctx).asDouble(); + } + long end = System.nanoTime(); + + out.millis = (end - begin) / (1000.0 * 1000.0); + } + + private static void getFeatures(ExpressionNode node, List<String> out) { + if (node instanceof ReferenceNode) { + String feature = ((ReferenceNode)node).getName(); + if (!out.contains(feature)) { + out.add(feature); + } + } else if (node instanceof CompositeNode) { + CompositeNode cNode = (CompositeNode)node; + for (ExpressionNode child : cNode.children()) { + getFeatures(child, out); + } + } + } + + private static class Result { + String name = "anonymous"; + double millis = Double.MAX_VALUE; + double res = 0; + } +} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java index 0e27d53338a..620da92ef95 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java @@ -1,79 +1,79 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.treenet;
-
-import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.searchlib.treenet.parser.ParseException;
-import com.yahoo.searchlib.treenet.parser.TreeNetParser;
-import junit.framework.TestCase;
-
-import java.io.*;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class TreeNetParserTestCase extends TestCase {
-
- private static final boolean WRITE_FILES = false;
-
- public void testRankingExpression() {
- for (int i = 1; i <= 8; ++i) {
- String inputFile = String.format("src/test/files/treenet%02d.model", i);
- String outputFile = String.format("src/test/files/ranking%02d.expression", i);
- String input = readFile(inputFile);
- String expression = convertModel(inputFile, input);
- if (WRITE_FILES) {
- writeFile(outputFile, expression);
- }
- else {
- String output = readFile(outputFile);
- assertParseable(output, outputFile);
- assertEquals(output.trim(), expression);
- }
- }
- }
-
- private void assertParseable(String rankingExpressionString,String fileName) {
- try {
- new RankingExpression(rankingExpressionString);
- }
- catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
- throw new RuntimeException("Could not parse ranking expression in '" + fileName + "'",e);
- }
- }
-
- private String convertModel(String modelFile, String model) {
- try {
- TreeNetParser parser = new TreeNetParser(new StringReader(model));
- return parser.treeNet().toRankingExpression();
- } catch (ParseException e) {
- throw new AssertionError("In model " + modelFile + ": " + e.getMessage(), e);
- }
- }
-
- private String readFile(String file) {
- try {
- StringBuilder ret = new StringBuilder();
- BufferedReader in = new BufferedReader(new FileReader(file));
- while (true) {
- String str = in.readLine();
- if (str == null) {
- break;
- }
- ret.append(str).append("\n");
- }
- return ret.toString();
- } catch (IOException e) {
- throw new AssertionError(e);
- }
- }
-
- private void writeFile(String file, String content) {
- try {
- FileWriter out = new FileWriter(file);
- out.write(content);
- out.close();
- } catch (IOException e) {
- throw new AssertionError(e);
- }
- }
-}
+package com.yahoo.searchlib.treenet; + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.treenet.parser.ParseException; +import com.yahoo.searchlib.treenet.parser.TreeNetParser; +import junit.framework.TestCase; + +import java.io.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TreeNetParserTestCase extends TestCase { + + private static final boolean WRITE_FILES = false; + + public void testRankingExpression() { + for (int i = 1; i <= 8; ++i) { + String inputFile = String.format("src/test/files/treenet%02d.model", i); + String outputFile = String.format("src/test/files/ranking%02d.expression", i); + String input = readFile(inputFile); + String expression = convertModel(inputFile, input); + if (WRITE_FILES) { + writeFile(outputFile, expression); + } + else { + String output = readFile(outputFile); + assertParseable(output, outputFile); + assertEquals(output.trim(), expression); + } + } + } + + private void assertParseable(String rankingExpressionString,String fileName) { + try { + new RankingExpression(rankingExpressionString); + } + catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { + throw new RuntimeException("Could not parse ranking expression in '" + fileName + "'",e); + } + } + + private String convertModel(String modelFile, String model) { + try { + TreeNetParser parser = new TreeNetParser(new StringReader(model)); + return parser.treeNet().toRankingExpression(); + } catch (ParseException e) { + throw new AssertionError("In model " + modelFile + ": " + e.getMessage(), e); + } + } + + private String readFile(String file) { + try { + StringBuilder ret = new StringBuilder(); + BufferedReader in = new BufferedReader(new FileReader(file)); + while (true) { + String str = in.readLine(); + if (str == null) { + break; + } + ret.append(str).append("\n"); + } + return ret.toString(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private void writeFile(String file, String content) { + try { + FileWriter out = new FileWriter(file); + out.write(content); + out.close(); + } catch (IOException e) { + throw new AssertionError(e); + } + } +} diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 3621537c8fd..e38837c8630 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -415,4 +415,4 @@ TEST_F("onSerializeForDescendingSort() is forwarded to target vector", Serialize } // attribute } // search -TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index f50767d22b3..a56d8285129 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -336,4 +336,4 @@ TEST_F("queryTerm() returns term context was created with", WsetValueFixture) { } // attribute } // search -TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index f5f1f97d29b..e835448da9e 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -477,4 +477,4 @@ Test::Main() } -TEST_APPHOOK(search::diskindex::Test);
\ No newline at end of file +TEST_APPHOOK(search::diskindex::Test); diff --git a/searchlib/src/tests/stackdumpiterator/testowner.ATS b/searchlib/src/tests/stackdumpiterator/testowner.ATS deleted file mode 100644 index 6d03b0836a4..00000000000 --- a/searchlib/src/tests/stackdumpiterator/testowner.ATS +++ /dev/null @@ -1 +0,0 @@ -vlarsen diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 6abf0ac5f27..1908d22d7b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -579,10 +579,10 @@ AttributeVector::SearchContext:: createIterator(fef::TermFieldMatchData *matchData, bool strict) { if (_plsc != NULL) { - SearchIterator::UP res = - _plsc->createPostingIterator(matchData, strict); - if (res.get() != NULL) + SearchIterator::UP res = _plsc->createPostingIterator(matchData, strict); + if (res) { return res; + } } return createFilterIterator(matchData, strict); } diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.cpp b/searchlib/src/vespa/searchlib/attribute/changevector.cpp index 912d6884279..b31e29bad63 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/changevector.cpp @@ -23,4 +23,4 @@ template class ChangeVectorT<ChangeTemplate<StringChangeData>>; template class ChangeVectorT<ChangeTemplate<NumericChangeData<int64_t>>>; template class ChangeVectorT<ChangeTemplate<NumericChangeData<double>>>; -}
\ No newline at end of file +} diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index 2960d573098..fe96928dccd 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -7,8 +7,8 @@ #include <vespa/searchlib/datastore/datastore.h> #include <vespa/searchlib/util/memoryusage.h> #include <vespa/vespalib/util/array.h> -#include <set> #include <vespa/searchlib/btree/btree.h> +#include <set> #include <atomic> namespace vespalib { class asciistream; } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h index 556ce7c2722..e07aa421c3a 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h @@ -93,4 +93,4 @@ private: }; } // attribute -} // search
\ No newline at end of file +} // search diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 51b765f725a..c117cdc66c5 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -68,4 +68,4 @@ bool ImportedSearchContext::cmp(DocId docId) const { } } // attribute -} // search
\ No newline at end of file +} // search diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index c972e25a7cf..0b7430599b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -4,6 +4,7 @@ #include "multivalue.h" #include "multi_value_mapping.h" #include "postinglistattribute.h" +#include <vespa/searchlib/common/growablebitvector.h> #include <vespa/vespalib/util/array.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 9665872f8de..cb3dd844041 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -6,13 +6,11 @@ #include "postinglisttraits.h" #include "postingstore.h" #include "ipostinglistsearchcontext.h" -#include "attributevector.h" +#include <vespa/searchcommon/attribute/search_context_params.h> +#include <vespa/searchcommon/common/range.h> #include <vespa/vespalib/util/regexp.h> -#include <cstdlib> -namespace search { - -namespace attribute { +namespace search::attribute { /** * Search context helper for posting list attributes, used to instantiate @@ -22,11 +20,11 @@ namespace attribute { class PostingListSearchContext : public IPostingListSearchContext { protected: - typedef EnumPostingTree Dictionary; - typedef Dictionary::ConstIterator DictionaryConstIterator; - typedef Dictionary::FrozenView FrozenDictionary; - typedef EnumStoreBase::Index EnumIndex; - + using Dictionary = EnumPostingTree; + using DictionaryConstIterator = Dictionary::ConstIterator; + using FrozenDictionary = Dictionary::FrozenView; + using EnumIndex = EnumStoreBase::Index; + const FrozenDictionary _frozenDictionary; DictionaryConstIterator _lowerDictItr; DictionaryConstIterator _upperDictItr; @@ -36,22 +34,17 @@ protected: uint64_t _numValues; // attr.getStatus().getNumValues(); bool _hasWeight; bool _useBitVector; - search::datastore::EntryRef _pidx; - search::datastore::EntryRef _frozenRoot; // Posting list in tree form + datastore::EntryRef _pidx; + datastore::EntryRef _frozenRoot; // Posting list in tree form float _FSTC; // Filtering Search Time Constant float _PLSTC; // Posting List Search Time Constant const EnumStoreBase &_esb; uint32_t _minBvDocFreq; const GrowableBitVector *_gbv; // bitvector if _useBitVector has been set - - PostingListSearchContext(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const EnumStoreBase &esb, - uint32_t minBvDocFreq, - bool useBitVector); + + PostingListSearchContext(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, + const EnumStoreBase &esb, uint32_t minBvDocFreq, bool useBitVector); ~PostingListSearchContext(); @@ -96,13 +89,13 @@ template <class DataT> class PostingListSearchContextT : public PostingListSearchContext { protected: - typedef DataT DataType; - typedef PostingListTraits<DataType> Traits; - typedef typename Traits::PostingList PostingList; - typedef typename Traits::Posting Posting; - typedef std::vector<Posting> PostingVector; - typedef datastore::EntryRef EntryRef; - typedef typename PostingList::ConstIterator PostingConstIterator; + using DataType = DataT; + using Traits = PostingListTraits<DataType>; + using PostingList = typename Traits::PostingList; + using Posting = typename Traits::Posting; + using PostingVector = std::vector<Posting>; + using EntryRef = datastore::EntryRef; + using FrozenView = typename PostingList::BTreeType::FrozenView; const PostingList &_postingList; /* @@ -112,19 +105,14 @@ protected: std::unique_ptr<BitVector> _bitVector; bool _fetchPostingsDone; bool _arrayValid; - + static const long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; static const long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; static const long MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 10; - PostingListSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvCocFreq, - bool useBitVector); + PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, + bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvCocFreq, bool useBitVector); ~PostingListSearchContextT(); void lookupSingle(); @@ -133,16 +121,14 @@ protected: void fillBitVector(); PostingVector & - merge(PostingVector &v, PostingVector &temp, - const std::vector<size_t> & startPos) __attribute__((noinline)); + merge(PostingVector &v, PostingVector &temp, const std::vector<size_t> & startPos) __attribute__((noinline)); void fetchPostings(bool strict) override; // this will be called instead of the fetchPostings function in some cases - void diversify(bool forward, size_t wanted_hits, - const IAttributeVector &diversity_attr, size_t max_per_group, - size_t cutoff_groups, bool cutoff_strict); + void diversify(bool forward, size_t wanted_hits, const IAttributeVector &diversity_attr, + size_t max_per_group, size_t cutoff_groups, bool cutoff_strict); - queryeval::SearchIterator::UP + std::unique_ptr<queryeval::SearchIterator> createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) override; unsigned int singleHits() const; @@ -155,9 +141,9 @@ template <class DataT> class PostingListFoldedSearchContextT : public PostingListSearchContextT<DataT> { protected: - typedef PostingListSearchContextT<DataT> Parent; - typedef typename Parent::Dictionary Dictionary; - typedef typename Parent::PostingList PostingList; + using Parent = PostingListSearchContextT<DataT>; + using Dictionary = typename Parent::Dictionary; + using PostingList = typename Parent::PostingList; using Parent::_lowerDictItr; using Parent::_uniqueValues; using Parent::_postingList; @@ -165,14 +151,9 @@ protected: using Parent::countHits; using Parent::singleHits; - PostingListFoldedSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvCocFreq, - bool useBitVector); + PostingListFoldedSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, + bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvCocFreq, bool useBitVector); unsigned int approximateHits() const override; }; @@ -183,12 +164,12 @@ class PostingSearchContext: public BaseSC, public BaseSC2 { public: - typedef typename AttrT::EnumStore EnumStore; + using EnumStore = typename AttrT::EnumStore; using QueryTermSimpleUP = std::unique_ptr<QueryTermSimple>; protected: const AttrT &_toBeSearched; const EnumStore &_enumStore; - + PostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const AttrT &toBeSearched); ~PostingSearchContext(); }; @@ -198,14 +179,11 @@ class StringPostingSearchContext : public PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT> { private: - typedef PostingListTraits<DataT> AggregationTraits; - typedef typename AggregationTraits::PostingList PostingList; - typedef typename PostingList::Iterator PostingIterator; - typedef typename PostingList::ConstIterator PostingConstIterator; - typedef PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT> Parent; - typedef typename Parent::EnumStore EnumStore; - typedef typename EnumStore::FoldedComparatorType FoldedComparatorType; - typedef vespalib::Regexp Regexp; + using AggregationTraits = PostingListTraits<DataT>; + using PostingList = typename AggregationTraits::PostingList; + using Parent = PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT>; + using FoldedComparatorType = typename Parent::EnumStore::FoldedComparatorType; + using Regexp = vespalib::Regexp; using QueryTermSimpleUP = typename Parent::QueryTermSimpleUP; using Parent::_toBeSearched; using Parent::_enumStore; @@ -225,10 +203,7 @@ private: typedef PostingSearchContext<BaseSC, PostingListSearchContextT<DataT>, AttrT> Parent; typedef PostingListTraits<DataT> AggregationTraits; typedef typename AggregationTraits::PostingList PostingList; - typedef typename PostingList::Iterator PostingIterator; - typedef typename PostingList::ConstIterator PostingConstIterator; - typedef typename Parent::EnumStore EnumStore; - typedef typename EnumStore::ComparatorType ComparatorType; + typedef typename Parent::EnumStore::ComparatorType ComparatorType; typedef typename AttrT::T BaseType; using Params = attribute::SearchContextParams; using QueryTermSimpleUP = typename Parent::QueryTermSimpleUP; @@ -237,7 +212,7 @@ private: using Parent::_toBeSearched; using Parent::_enumStore; Params _params; - + void getIterators(bool shouldApplyRangeLimit); bool valid() const override { return this->isValid(); } @@ -269,8 +244,8 @@ public: NumericPostingSearchContext(QueryTermSimpleUP qTerm, const Params & params, const AttrT &toBeSearched); const Params ¶ms() const { return _params; } }; - - + + template <typename BaseSC, typename BaseSC2, typename AttrT> PostingSearchContext<BaseSC, BaseSC2, AttrT>:: PostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const AttrT &toBeSearched) @@ -305,7 +280,7 @@ StringPostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const Att // after benchmarking prefix search performance on single, array, and weighted set fast-search string attributes // with 1M values the following constant has been derived: this->_PLSTC = 0.000000; - + if (this->valid()) { if (this->isPrefix()) { FoldedComparatorType comp(_enumStore, this->queryTerm().getTerm(), true); @@ -386,7 +361,4 @@ extern template class PostingListSearchContextT<int32_t>; extern template class PostingListFoldedSearchContextT<btree::BTreeNoLeafData>; extern template class PostingListFoldedSearchContextT<int32_t>; -} // namespace attribute - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 270ae161f4c..573af5f40bd 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -5,28 +5,23 @@ #include "dociditerator.h" #include "attributeiterators.h" #include "diversity.h" +#include "postingstore.hpp" #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/searchlib/common/growablebitvector.h> -namespace search { -using queryeval::EmptySearch; -using queryeval::SearchIterator; +using search::queryeval::EmptySearch; +using search::queryeval::SearchIterator; -namespace attribute { +namespace search::attribute { template <typename DataT> PostingListSearchContextT<DataT>:: -PostingListSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvDocFreq, - bool useBitVector) - : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, - esb, minBvDocFreq, useBitVector), +PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, + const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvDocFreq, bool useBitVector) + : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, esb, minBvDocFreq, useBitVector), _postingList(postingList), _array(), _bitVector(), @@ -56,9 +51,7 @@ PostingListSearchContextT<DataT>::lookupSingle() } else { _pidx = bve->_tree; if (_pidx.valid()) { - typename PostingList::BTreeType::FrozenView - frozenView(_postingList.getTreeEntry(_pidx)-> - getFrozenView(_postingList.getAllocator())); + auto frozenView = _postingList.getTreeEntry(_pidx)->getFrozenView(_postingList.getAllocator()); _frozenRoot = frozenView.getRoot(); if (!_frozenRoot.valid()) { _pidx = datastore::EntryRef(); @@ -68,9 +61,7 @@ PostingListSearchContextT<DataT>::lookupSingle() } } } else { - typename PostingList::BTreeType::FrozenView - frozenView(_postingList.getTreeEntry(_pidx)-> - getFrozenView(_postingList.getAllocator())); + auto frozenView = _postingList.getTreeEntry(_pidx)->getFrozenView(_postingList.getAllocator()); _frozenRoot = frozenView.getRoot(); if (!_frozenRoot.valid()) { _pidx = datastore::EntryRef(); @@ -106,8 +97,8 @@ PostingListSearchContextT<DataT>::fillArray(size_t numDocs) for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { if (useThis(it)) { _postingList.foreach_frozen(it.getData(), - [&](uint32_t key, const DataT &data) - { _array.push_back(Posting(key, data)); + [&](uint32_t key, const DataT &data) { + _array.push_back(Posting(key, data)); }); startPos.push_back(_array.size()); } @@ -130,8 +121,8 @@ PostingListSearchContextT<DataT>::fillBitVector() for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { if (useThis(it)) { _postingList.foreach_frozen_key(it.getData(), - [&](uint32_t key) - { if (key < limit) { + [&](uint32_t key) { + if (key < limit) { bv.setBit(key); } }); @@ -193,9 +184,8 @@ PostingListSearchContextT<DataT>::fetchPostings(bool strict) template <typename DataT> void -PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, - const IAttributeVector &diversity_attr, size_t max_per_group, - size_t cutoff_groups, bool cutoff_strict) +PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, const IAttributeVector &diversity_attr, + size_t max_per_group, size_t cutoff_groups, bool cutoff_strict) { assert(!_fetchPostingsDone); _fetchPostingsDone = true; @@ -203,9 +193,8 @@ PostingListSearchContextT<DataT>::diversify(bool forward, size_t wanted_hits, _array.reserve(wanted_hits); std::vector<size_t> fragments; fragments.push_back(0); - diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, - diversity_attr, max_per_group, cutoff_groups, cutoff_strict, - _array, fragments); + diversity::diversify(forward, _lowerDictItr, _upperDictItr, _postingList, wanted_hits, diversity_attr, + max_per_group, cutoff_groups, cutoff_strict, _array, fragments); if (fragments.size() > 2) { PostingVector temp(_array.size()); _array.swap(merge(_array, temp, fragments)); @@ -226,12 +215,14 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) if (_arrayValid || (_bitVector.get() != nullptr)) { // synthetic results are available if (!_array.empty()) { assert(_arrayValid); - typedef DocIdIterator<Posting> DocIt; + using DocIt = DocIdIterator<Posting>; DocIt postings; postings.set(&_array[0], &_array[_array.size()]); - return (_postingList._isFilter) - ? SearchIterator::UP(new FilterAttributePostingListIteratorT<DocIt>(matchData, postings)) - : SearchIterator::UP(new AttributePostingListIteratorT<DocIt>(_hasWeight, matchData, postings)); + if (_postingList._isFilter) { + return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(matchData, postings); + } else { + return std::make_unique<AttributePostingListIteratorT<DocIt>>(_hasWeight, matchData, postings); + } } if (_arrayValid) { return SearchIterator::UP(new EmptySearch()); @@ -251,19 +242,24 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) if (!_frozenRoot.valid()) { uint32_t clusterSize = _postingList.getClusterSize(_pidx); assert(clusterSize != 0); - typedef DocIdMinMaxIterator<Posting> DocIt; + using DocIt = DocIdMinMaxIterator<Posting>; DocIt postings; const Posting *array = postingList.getKeyDataEntry(_pidx, clusterSize); postings.set(array, array + clusterSize); - return (postingList._isFilter) - ? SearchIterator::UP(new FilterAttributePostingListIteratorT<DocIt>(matchData, postings)) - : SearchIterator::UP(new AttributePostingListIteratorT<DocIt>(_hasWeight, matchData, postings)); + if (postingList._isFilter) { + return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(matchData, postings); + } else { + return std::make_unique<AttributePostingListIteratorT<DocIt>>(_hasWeight, matchData, postings); + } } typename PostingList::BTreeType::FrozenView frozen(_frozenRoot, postingList.getAllocator()); - return (_postingList._isFilter) - ? SearchIterator::UP(new FilterAttributePostingListIteratorT<PostingConstIterator> (matchData, frozen.getRoot(), frozen.getAllocator())) - : SearchIterator::UP(new AttributePostingListIteratorT<PostingConstIterator> (_hasWeight, matchData, frozen.getRoot(), frozen.getAllocator())); + using DocIt = typename PostingList::ConstIterator; + if (_postingList._isFilter) { + return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(matchData, frozen.getRoot(), frozen.getAllocator()); + } else { + return std::make_unique<AttributePostingListIteratorT<DocIt>> (_hasWeight, matchData, frozen.getRoot(), frozen.getAllocator()); + } } // returning nullptr will trigger fallback to filter iterator return SearchIterator::UP(); @@ -300,12 +296,10 @@ PostingListSearchContextT<DataT>::approximateHits() const if (this->fallbackToFiltering()) { numHits = _docIdLimit; } else if (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION) { - if ((_uniqueValues * - MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > - static_cast<int>(_docIdLimit)) || - (this->calculateApproxNumHits() * - MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > - _docIdLimit)) { + if ((_uniqueValues * MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > static_cast<int>(_docIdLimit)) || + (this->calculateApproxNumHits() * MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > _docIdLimit) || + (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION*10)) + { numHits = this->calculateApproxNumHits(); } else { // XXX: Unsafe @@ -346,16 +340,10 @@ PostingListSearchContextT<DataT>::applyRangeLimit(int rangeLimit) template <typename DataT> PostingListFoldedSearchContextT<DataT>:: -PostingListFoldedSearchContextT(const Dictionary &dictionary, - uint32_t docIdLimit, - uint64_t numValues, - bool hasWeight, - const PostingList &postingList, - const EnumStoreBase &esb, - uint32_t minBvDocFreq, - bool useBitVector) - : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, - esb, minBvDocFreq, useBitVector) +PostingListFoldedSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, + bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + uint32_t minBvDocFreq, bool useBitVector) + : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, esb, minBvDocFreq, useBitVector) { } @@ -379,7 +367,4 @@ PostingListFoldedSearchContextT<DataT>::approximateHits() const return numHits; } -} // namespace attribute - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h index c2504f779a1..d619751d451 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglisttraits.h @@ -4,11 +4,9 @@ #include <vespa/searchlib/btree/btreestore.h> -namespace search -{ +namespace search { -namespace attribute -{ +namespace attribute { template <typename DataT> class PostingListTraits; template <typename DataT> class PostingStore; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index a21884bca4a..e1b0cf24305 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -3,12 +3,11 @@ #include "postingstore.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> +#include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcommon/attribute/status.h> -namespace search { - -namespace attribute { +namespace search::attribute { using btree::BTreeNoLeafData; @@ -635,6 +634,4 @@ template class PostingStore<BTreeNoLeafData>; template class PostingStore<int32_t>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 9aae723db73..9c34344451a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -5,12 +5,13 @@ #include "postinglisttraits.h" #include "enumstorebase.h" #include <set> -#include <vespa/searchlib/common/bitvector.h> -#include <vespa/searchlib/common/growablebitvector.h> namespace search { + class BitVector; + class GrowableBitVector; +} -namespace attribute { +namespace search::attribute { class Status; class Config; @@ -43,9 +44,9 @@ public: uint32_t _maxBvDocFreq; // Greater than or equal to this ==> create bv protected: std::set<uint32_t> _bvs; // Current bitvectors - EnumPostingTree &_dict; - Status &_status; - uint64_t _bvExtraBytes; + EnumPostingTree &_dict; + Status &_status; + uint64_t _bvExtraBytes; static constexpr uint32_t BUFFERTYPE_BITVECTOR = 9u; @@ -197,90 +198,4 @@ PostingStore<int32_t>::bitVectorWeight() return 1; } -template <typename DataT> -template <typename FunctionType> -void -PostingStore<DataT>::foreach_frozen_key(EntryRef ref, FunctionType func) const -{ - if (!ref.valid()) - return; - RefType iRef(ref); - uint32_t typeId = getTypeId(iRef); - uint32_t clusterSize = getClusterSize(typeId); - if (clusterSize == 0) { - if (isBitVector(typeId)) { - const BitVectorEntry *bve = getBitVectorEntry(iRef); - EntryRef ref2(bve->_tree); - RefType iRef2(ref2); - if (iRef2.valid()) { - assert(isBTree(iRef2)); - const BTreeType *tree = getTreeEntry(iRef2); - _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); - } else { - const BitVector *bv = bve->_bv.get(); - uint32_t docIdLimit = bv->size(); - uint32_t docId = bv->getFirstTrueBit(1); - while (docId < docIdLimit) { - func(docId); - docId = bv->getNextTrueBit(docId + 1); - } - } - } else { - assert(isBTree(typeId)); - const BTreeType *tree = getTreeEntry(iRef); - _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); - } - } else { - const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); - const KeyDataType *pe = p + clusterSize; - for (; p != pe; ++p) { - func(p->_key); - } - } -} - - -template <typename DataT> -template <typename FunctionType> -void -PostingStore<DataT>::foreach_frozen(EntryRef ref, FunctionType func) const -{ - if (!ref.valid()) - return; - RefType iRef(ref); - uint32_t typeId = getTypeId(iRef); - uint32_t clusterSize = getClusterSize(typeId); - if (clusterSize == 0) { - if (isBitVector(typeId)) { - const BitVectorEntry *bve = getBitVectorEntry(iRef); - EntryRef ref2(bve->_tree); - RefType iRef2(ref2); - if (iRef2.valid()) { - assert(isBTree(iRef2)); - const BTreeType *tree = getTreeEntry(iRef2); - _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); - } else { - const BitVector *bv = bve->_bv.get(); - uint32_t docIdLimit = bv->size(); - uint32_t docId = bv->getFirstTrueBit(1); - while (docId < docIdLimit) { - func(docId, bitVectorWeight()); - docId = bv->getNextTrueBit(docId + 1); - } - } - } else { - const BTreeType *tree = getTreeEntry(iRef); - _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); - } - } else { - const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); - const KeyDataType *pe = p + clusterSize; - for (; p != pe; ++p) { - func(p->_key, p->getData()); - } - } } - -} // namespace attribute - -} // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.hpp b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp new file mode 100644 index 00000000000..6680fc86f71 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp @@ -0,0 +1,90 @@ +#pragma once + +#include "postingstore.h" +#include <vespa/searchlib/common/growablebitvector.h> + +namespace search::attribute { + +template<typename DataT> +template<typename FunctionType> +void +PostingStore<DataT>::foreach_frozen_key(EntryRef ref, FunctionType func) const { + if (!ref.valid()) + return; + RefType iRef(ref); + uint32_t typeId = getTypeId(iRef); + uint32_t clusterSize = getClusterSize(typeId); + if (clusterSize == 0) { + if (isBitVector(typeId)) { + const BitVectorEntry *bve = getBitVectorEntry(iRef); + EntryRef ref2(bve->_tree); + RefType iRef2(ref2); + if (iRef2.valid()) { + assert(isBTree(iRef2)); + const BTreeType *tree = getTreeEntry(iRef2); + _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); + } else { + const BitVector *bv = bve->_bv.get(); + uint32_t docIdLimit = bv->size(); + uint32_t docId = bv->getFirstTrueBit(1); + while (docId < docIdLimit) { + func(docId); + docId = bv->getNextTrueBit(docId + 1); + } + } + } else { + assert(isBTree(typeId)); + const BTreeType *tree = getTreeEntry(iRef); + _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); + } + } else { + const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); + const KeyDataType *pe = p + clusterSize; + for (; p != pe; ++p) { + func(p->_key); + } + } +} + + +template<typename DataT> +template<typename FunctionType> +void +PostingStore<DataT>::foreach_frozen(EntryRef ref, FunctionType func) const { + if (!ref.valid()) + return; + RefType iRef(ref); + uint32_t typeId = getTypeId(iRef); + uint32_t clusterSize = getClusterSize(typeId); + if (clusterSize == 0) { + if (isBitVector(typeId)) { + const BitVectorEntry *bve = getBitVectorEntry(iRef); + EntryRef ref2(bve->_tree); + RefType iRef2(ref2); + if (iRef2.valid()) { + assert(isBTree(iRef2)); + const BTreeType *tree = getTreeEntry(iRef2); + _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); + } else { + const BitVector *bv = bve->_bv.get(); + uint32_t docIdLimit = bv->size(); + uint32_t docId = bv->getFirstTrueBit(1); + while (docId < docIdLimit) { + func(docId, bitVectorWeight()); + docId = bv->getNextTrueBit(docId + 1); + } + } + } else { + const BTreeType *tree = getTreeEntry(iRef); + _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); + } + } else { + const KeyDataType *p = getKeyDataEntry(iRef, clusterSize); + const KeyDataType *pe = p + clusterSize; + for (; p != pe; ++p) { + func(p->_key, p->getData()); + } + } +} + +} diff --git a/searchlib/src/vespa/searchlib/btree/btreestore.h b/searchlib/src/vespa/searchlib/btree/btreestore.h index e8153f94363..c22d2dd3f55 100644 --- a/searchlib/src/vespa/searchlib/btree/btreestore.h +++ b/searchlib/src/vespa/searchlib/btree/btreestore.h @@ -10,11 +10,7 @@ #include <vespa/searchlib/datastore/datastore.h> #include <vespa/searchlib/datastore/handle.h> -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, @@ -508,8 +504,6 @@ extern template class BTreeStore<uint32_t, int32_t, BTreeDefaultTraits, MinMaxAggrCalc>; -} // namespace btree - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/datastore/datastore.cpp b/searchlib/src/vespa/searchlib/datastore/datastore.cpp index 13b3bf3a6e2..14edb5b5cd5 100644 --- a/searchlib/src/vespa/searchlib/datastore/datastore.cpp +++ b/searchlib/src/vespa/searchlib/datastore/datastore.cpp @@ -16,4 +16,4 @@ template class DataStoreT<EntryRefT<22> >; template void vespalib::Array<search::datastore::DataStoreBase::ElemHold1ListElem>::increase(size_t); template class search::attribute::RcuVector<search::datastore::EntryRef>; template class search::attribute::RcuVectorBase<search::datastore::EntryRef>; -//template void search::attribute::RcuVectorBase<search::datastore::EntryRef>::expandAndInsert(const search::datastore::EntryRef &);
\ No newline at end of file +//template void search::attribute::RcuVectorBase<search::datastore::EntryRef>::expandAndInsert(const search::datastore::EntryRef &); diff --git a/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp b/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp index c24b4e25821..b850ddf79cc 100644 --- a/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp +++ b/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp @@ -33,4 +33,4 @@ PropertiesMap::lookup(const vespalib::stringref &name) const } // namespace engine } // namespace search -VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, search::fef::Properties);
\ No newline at end of file +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, search::fef::Properties); diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h index eb278280156..642b942390a 100644 --- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h +++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h @@ -261,4 +261,4 @@ bool has_active_enum_guards(AttributeVector &attr) { } } // attribute -} // search
\ No newline at end of file +} // search diff --git a/searchlib/src/vespa/searchlib/test/statestring.h b/searchlib/src/vespa/searchlib/test/statestring.h index 7fb6f1fce85..475872b8e89 100644 --- a/searchlib/src/vespa/searchlib/test/statestring.h +++ b/searchlib/src/vespa/searchlib/test/statestring.h @@ -10,4 +10,4 @@ void normalizeAddr(vespalib::string &s, void *addr); void normalizeTimestamps(std::vector<vespalib::string> &sv); void normalizeAddrs(std::vector<vespalib::string> &sv, void *addr); -}
\ No newline at end of file +} diff --git a/searchlib/src/vespa/searchlib/util/url.h b/searchlib/src/vespa/searchlib/util/url.h index 88a4ef6d9f7..35de9d2206c 100644 --- a/searchlib/src/vespa/searchlib/util/url.h +++ b/searchlib/src/vespa/searchlib/util/url.h @@ -272,4 +272,4 @@ public: void Dump(); }; -}
\ No newline at end of file +} diff --git a/searchsummary/src/tests/extractkeywords/testowner.ATS b/searchsummary/src/tests/extractkeywords/testowner.ATS deleted file mode 100644 index 6d03b0836a4..00000000000 --- a/searchsummary/src/tests/extractkeywords/testowner.ATS +++ /dev/null @@ -1 +0,0 @@ -vlarsen diff --git a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java index 4122e9f1bd8..772419c54bb 100644 --- a/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java +++ b/serviceview/src/main/java/com/yahoo/vespa/serviceview/bindings/ClusterView.java @@ -17,4 +17,4 @@ public class ClusterView { @JsonInclude(value = Include.NON_NULL) public String url; public List<ServiceView> services; -}
\ No newline at end of file +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala index 443806d014b..15fa3aa38e4 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala @@ -23,4 +23,4 @@ object Converter { implicit val javaBooleanConverter:Converter[java.lang.Boolean] = toConverter(_.toBoolean) -}
\ No newline at end of file +} diff --git a/storage/src/vespa/storage/distributor/operation_sequencer.h b/storage/src/vespa/storage/distributor/operation_sequencer.h index 2a00a237bd5..1cf11a784f8 100644 --- a/storage/src/vespa/storage/distributor/operation_sequencer.h +++ b/storage/src/vespa/storage/distributor/operation_sequencer.h @@ -82,4 +82,4 @@ private: void release(const SequencingHandle& handle); }; -} // storage::distributor
\ No newline at end of file +} // storage::distributor diff --git a/storage/src/vespa/storage/distributor/operations/sequenced_operation.h b/storage/src/vespa/storage/distributor/operations/sequenced_operation.h index c4b37eba9e0..8a5c9439135 100644 --- a/storage/src/vespa/storage/distributor/operations/sequenced_operation.h +++ b/storage/src/vespa/storage/distributor/operations/sequenced_operation.h @@ -22,4 +22,4 @@ public: } }; -} // storage::distributor
\ No newline at end of file +} // storage::distributor diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index f84f9d38386..8978c504a41 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -189,4 +189,4 @@ AbortBucketOperationsCommand::makeReply() { return std::make_unique<AbortBucketOperationsReply>(*this); } -}
\ No newline at end of file +} diff --git a/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java b/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java index c1cb8e4032c..77ee9e0f457 100644 --- a/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java +++ b/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java @@ -40,4 +40,4 @@ public class LinePatternMatcher extends BaseMatcher<String> { return new LinePatternMatcher(pattern); } -}
\ No newline at end of file +} diff --git a/testutil/src/main/java/com/yahoo/test/PatternMatcher.java b/testutil/src/main/java/com/yahoo/test/PatternMatcher.java index 84cc94e32a2..643428f6b92 100644 --- a/testutil/src/main/java/com/yahoo/test/PatternMatcher.java +++ b/testutil/src/main/java/com/yahoo/test/PatternMatcher.java @@ -35,4 +35,4 @@ public class PatternMatcher extends BaseMatcher<String> { return new PatternMatcher(pattern); } -}
\ No newline at end of file +} diff --git a/vbench/src/tests/app_vbench/CMakeLists.txt b/vbench/src/tests/app_vbench/CMakeLists.txt index c5e57a8505e..a8af166489c 100644 --- a/vbench/src/tests/app_vbench/CMakeLists.txt +++ b/vbench/src/tests/app_vbench/CMakeLists.txt @@ -7,4 +7,4 @@ vespa_add_executable(vbench_app_vbench_test_app TEST vbench ) vespa_add_test(NAME vbench_app_vbench_test_app NO_VALGRIND COMMAND vbench_app_vbench_test_app) -configure_file(vbench.cfg.template vbench.cfg @ONLY)
\ No newline at end of file +configure_file(vbench.cfg.template vbench.cfg @ONLY) diff --git a/vdslib/pom.xml b/vdslib/pom.xml index a4ada46c89e..26d210f14dc 100644 --- a/vdslib/pom.xml +++ b/vdslib/pom.xml @@ -99,7 +99,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-install-plugin</artifactId> - <version>2.3.1</version> <configuration> <updateReleaseInfo>true</updateReleaseInfo> </configuration> diff --git a/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java b/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java index 0e3e80b2542..c49109762e6 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java @@ -1,205 +1,205 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib;
-
-import com.yahoo.document.BucketId;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class BucketDistribution {
-
- // A logger object to enable proper logging.
- private static Logger log = Logger.getLogger(BucketDistribution.class.getName());
-
- // A map from bucket id to column index.
- private int[] bucketToColumn;
-
- // The number of columns to distribute to.
- private int numColumns;
-
- // The number of bits to use for bucket identification.
- private int numBucketBits;
-
- /**
- * Constructs a new bucket distribution object with a given number of columns and buckets.
- *
- * @param numColumns The number of columns to distribute to.
- * @param numBucketBits The number of bits to use for bucket id.
- */
- public BucketDistribution(int numColumns, int numBucketBits) {
- this.numBucketBits = numBucketBits;
- bucketToColumn = new int[getNumBuckets()];
- reset();
- setNumColumns(numColumns);
- }
-
- /**
- * Constructs a new bucket distribution object as a copy of another.
- *
- * @param other The distribution object to copy.
- */
- public BucketDistribution(BucketDistribution other) {
- bucketToColumn = other.bucketToColumn.clone();
- numColumns = other.numColumns;
- numBucketBits = other.numBucketBits;
- }
-
- /**
- * Returns the number of buckets that the given number of bucket bits will allow.
- *
- * @param numBucketBits The number of bits to use for bucket id.
- * @return The number of buckets allowed.
- */
- private static int getNumBuckets(int numBucketBits) {
- return 1 << numBucketBits;
- }
-
- /**
- * This method returns a list that contains the distributions of the given number of buckets over the given number
- * of columns.
- *
- * @param numColumns The number of columns to distribute to.
- * @param numBucketBits The number of bits to use for bucket id.
- * @return The bucket distribution.
- */
- private static List<Integer> getBucketCount(int numColumns, int numBucketBits) {
- List<Integer> ret = new ArrayList<Integer>(numColumns);
- int cnt = getNumBuckets(numBucketBits) / numColumns;
- int rst = getNumBuckets(numBucketBits) % numColumns;
- for (int i = 0; i < numColumns; ++i) {
- ret.add(cnt + (i < rst ? 1 : 0));
- }
- return ret;
- }
-
- /**
- * This method returns a list similar to {@link com.yahoo.vdslib.BucketDistribution#getBucketCount(int, int)}, except that the returned list
- * contains the number of buckets that will have to be migrated from each column if an additional column was added.
- *
- * @param numColumns The original number of columns.
- * @param numBucketBits The number of bits to use for bucket id.
- * @return The number of buckets to migrate, one value per column.
- */
- private static List<Integer> getBucketMigrateCount(int numColumns, int numBucketBits) {
- List<Integer> ret = getBucketCount(numColumns++, numBucketBits);
- int cnt = getNumBuckets(numBucketBits) / numColumns;
- int rst = getNumBuckets(numBucketBits) % numColumns;
- for (int i = 0; i < numColumns - 1; ++i) {
- ret.set(i, ret.get(i) - (cnt + (i < rst ? 1 : 0)));
- }
- return ret;
- }
-
- /**
- * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so
- * that it all buckets point to that single column.
- */
- public void reset() {
- for (int i = 0; i < bucketToColumn.length; ++i) {
- bucketToColumn[i] = 0;
- }
- numColumns = 1;
- }
-
- /**
- * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so
- * that it takes into account the new column.
- */
- private void addColumn() {
- int newColumns = numColumns + 1;
- List<Integer> migrate = getBucketMigrateCount(numColumns, numBucketBits);
- int numBuckets = getNumBuckets(numBucketBits);
- for (int i = 0; i < numBuckets; ++i) {
- int old = bucketToColumn[i];
- if (migrate.get(old) > 0) {
- bucketToColumn[i] = numColumns; // move this bucket to the new column
- migrate.set(old, migrate.get(old) - 1);
- }
- }
- numColumns = newColumns;
- }
-
- /**
- * Sets the number of columns to use for this document distribution object. This will reset and setup this object
- * from scratch. The original number of buckets is maintained.
- *
- * @param numColumns The new number of columns to distribute to.
- */
- public synchronized void setNumColumns(int numColumns) {
- if (numColumns < this.numColumns) {
- reset();
- }
- if (numColumns == this.numColumns) {
- return;
- }
- for (int i = numColumns - this.numColumns; --i >= 0; ) {
- addColumn();
- }
- }
-
- /**
- * Returns the number of columns to distribute to.
- *
- * @return The number of columns.
- */
- public int getNumColumns() {
- return numColumns;
- }
-
- /**
- * Sets the number of buckets to use for this document distribution object. This will reset and setup this object
- * from scratch. The original number of columns is maintained.
- *
- * @param numBucketBits The new number of bits to use for bucket id.
- */
- public synchronized void setNumBucketBits(int numBucketBits) {
- if (numBucketBits == this.numBucketBits) {
- return;
- }
- this.numBucketBits = numBucketBits;
- bucketToColumn = new int[getNumBuckets(numBucketBits)];
- int numColumns = this.numColumns;
- reset();
- setNumColumns(numColumns);
- }
-
- /**
- * Returns the number of bits used for bucket identifiers.
- *
- * @return The number of bits.
- */
- public int getNumBucketBits() {
- return numBucketBits;
- }
-
- /**
- * Returns the number of buckets available using the configured number of bucket bits.
- *
- * @return The number of buckets.
- */
- public int getNumBuckets() {
- return getNumBuckets(numBucketBits);
- }
-
- /**
- * This method maps the given bucket id to its corresponding column.
- *
- * @param bucketId The bucket whose column to lookup.
- * @return The column to distribute the bucket to.
- */
- public int getColumn(BucketId bucketId) {
- int ret = (int)(bucketId.getId() & (getNumBuckets(numBucketBits) - 1));
- if (ret >= bucketToColumn.length) {
- log.log(Level.SEVERE,
- "The bucket distribution map is not in sync with the number of bucket bits. " +
- "This should never happen! Distribution is broken!!");
- return 0;
- }
- return bucketToColumn[ret];
- }
-}
+package com.yahoo.vdslib; + +import com.yahoo.document.BucketId; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class BucketDistribution { + + // A logger object to enable proper logging. + private static Logger log = Logger.getLogger(BucketDistribution.class.getName()); + + // A map from bucket id to column index. + private int[] bucketToColumn; + + // The number of columns to distribute to. + private int numColumns; + + // The number of bits to use for bucket identification. + private int numBucketBits; + + /** + * Constructs a new bucket distribution object with a given number of columns and buckets. + * + * @param numColumns The number of columns to distribute to. + * @param numBucketBits The number of bits to use for bucket id. + */ + public BucketDistribution(int numColumns, int numBucketBits) { + this.numBucketBits = numBucketBits; + bucketToColumn = new int[getNumBuckets()]; + reset(); + setNumColumns(numColumns); + } + + /** + * Constructs a new bucket distribution object as a copy of another. + * + * @param other The distribution object to copy. + */ + public BucketDistribution(BucketDistribution other) { + bucketToColumn = other.bucketToColumn.clone(); + numColumns = other.numColumns; + numBucketBits = other.numBucketBits; + } + + /** + * Returns the number of buckets that the given number of bucket bits will allow. + * + * @param numBucketBits The number of bits to use for bucket id. + * @return The number of buckets allowed. + */ + private static int getNumBuckets(int numBucketBits) { + return 1 << numBucketBits; + } + + /** + * This method returns a list that contains the distributions of the given number of buckets over the given number + * of columns. + * + * @param numColumns The number of columns to distribute to. + * @param numBucketBits The number of bits to use for bucket id. + * @return The bucket distribution. + */ + private static List<Integer> getBucketCount(int numColumns, int numBucketBits) { + List<Integer> ret = new ArrayList<Integer>(numColumns); + int cnt = getNumBuckets(numBucketBits) / numColumns; + int rst = getNumBuckets(numBucketBits) % numColumns; + for (int i = 0; i < numColumns; ++i) { + ret.add(cnt + (i < rst ? 1 : 0)); + } + return ret; + } + + /** + * This method returns a list similar to {@link com.yahoo.vdslib.BucketDistribution#getBucketCount(int, int)}, except that the returned list + * contains the number of buckets that will have to be migrated from each column if an additional column was added. + * + * @param numColumns The original number of columns. + * @param numBucketBits The number of bits to use for bucket id. + * @return The number of buckets to migrate, one value per column. + */ + private static List<Integer> getBucketMigrateCount(int numColumns, int numBucketBits) { + List<Integer> ret = getBucketCount(numColumns++, numBucketBits); + int cnt = getNumBuckets(numBucketBits) / numColumns; + int rst = getNumBuckets(numBucketBits) % numColumns; + for (int i = 0; i < numColumns - 1; ++i) { + ret.set(i, ret.get(i) - (cnt + (i < rst ? 1 : 0))); + } + return ret; + } + + /** + * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so + * that it all buckets point to that single column. + */ + public void reset() { + for (int i = 0; i < bucketToColumn.length; ++i) { + bucketToColumn[i] = 0; + } + numColumns = 1; + } + + /** + * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so + * that it takes into account the new column. + */ + private void addColumn() { + int newColumns = numColumns + 1; + List<Integer> migrate = getBucketMigrateCount(numColumns, numBucketBits); + int numBuckets = getNumBuckets(numBucketBits); + for (int i = 0; i < numBuckets; ++i) { + int old = bucketToColumn[i]; + if (migrate.get(old) > 0) { + bucketToColumn[i] = numColumns; // move this bucket to the new column + migrate.set(old, migrate.get(old) - 1); + } + } + numColumns = newColumns; + } + + /** + * Sets the number of columns to use for this document distribution object. This will reset and setup this object + * from scratch. The original number of buckets is maintained. + * + * @param numColumns The new number of columns to distribute to. + */ + public synchronized void setNumColumns(int numColumns) { + if (numColumns < this.numColumns) { + reset(); + } + if (numColumns == this.numColumns) { + return; + } + for (int i = numColumns - this.numColumns; --i >= 0; ) { + addColumn(); + } + } + + /** + * Returns the number of columns to distribute to. + * + * @return The number of columns. + */ + public int getNumColumns() { + return numColumns; + } + + /** + * Sets the number of buckets to use for this document distribution object. This will reset and setup this object + * from scratch. The original number of columns is maintained. + * + * @param numBucketBits The new number of bits to use for bucket id. + */ + public synchronized void setNumBucketBits(int numBucketBits) { + if (numBucketBits == this.numBucketBits) { + return; + } + this.numBucketBits = numBucketBits; + bucketToColumn = new int[getNumBuckets(numBucketBits)]; + int numColumns = this.numColumns; + reset(); + setNumColumns(numColumns); + } + + /** + * Returns the number of bits used for bucket identifiers. + * + * @return The number of bits. + */ + public int getNumBucketBits() { + return numBucketBits; + } + + /** + * Returns the number of buckets available using the configured number of bucket bits. + * + * @return The number of buckets. + */ + public int getNumBuckets() { + return getNumBuckets(numBucketBits); + } + + /** + * This method maps the given bucket id to its corresponding column. + * + * @param bucketId The bucket whose column to lookup. + * @return The column to distribute the bucket to. + */ + public int getColumn(BucketId bucketId) { + int ret = (int)(bucketId.getId() & (getNumBuckets(numBucketBits) - 1)); + if (ret >= bucketToColumn.length) { + log.log(Level.SEVERE, + "The bucket distribution map is not in sync with the number of bucket bits. " + + "This should never happen! Distribution is broken!!"); + return 0; + } + return bucketToColumn[ret]; + } +} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java index 33b8dd39e55..1e9851a3788 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java @@ -1,111 +1,111 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib;
-
-import com.yahoo.document.BucketId;
-
-import java.util.Random;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class BucketDistributionTestCase extends junit.framework.TestCase {
-
- private static final int NUM_COLUMNS = 16;
- private static final int MIN_BUCKETBITS = 4;
- private static final int MAX_BUCKETBITS = 9;
-
- public void printExpected() {
- System.out.println(" int[][] expected = new int[][] {");
- for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) {
- System.out.print(" new int[] {");
- BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits);
- for (int i = 0; i < bd.getNumBuckets(); ++i) {
- if (i % 32 == 0) {
- System.out.println("");
- System.out.print(" ");
- }
- System.out.print(bd.getColumn(new BucketId(16, i)));
- if (i < bd.getNumBuckets() - 1) {
- System.out.print(", ");
- }
- }
- System.out.print(" }");
- if (numBucketBits < MAX_BUCKETBITS) {
- System.out.print(",");
- }
- System.out.println("");
- }
- System.out.println(" };");
- }
-
- public void testDistribution() {
- int[][] expected = new int[][] {
- new int[] {
- 10, 11, 9, 6, 4, 8, 14, 1, 13, 2, 12, 3, 5, 7, 15, 0 },
- new int[] {
- 11, 12, 11, 13, 8, 13, 8, 9, 4, 5, 6, 12, 10, 15, 1, 1, 7, 9, 14, 2, 2, 14, 3, 3, 4, 5, 6, 7, 10, 15, 0, 0 },
- new int[] {
- 13, 13, 13, 13, 9, 11, 8, 9, 11, 14, 9, 11, 14, 14, 8, 10, 11, 14, 4, 5, 5, 6, 6, 7, 8, 10, 12, 15, 1, 1, 1, 1,
- 6, 7, 8, 10, 12, 15, 2, 2, 2, 2, 12, 15, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7, 7, 9, 10, 12, 15, 0, 0, 0, 0 },
- new int[] {
- 14, 14, 14, 13, 11, 14, 14, 10, 13, 14, 10, 12, 8, 8, 9, 10, 9, 10, 11, 12, 13, 15, 11, 12, 13, 13, 15, 8, 8, 9, 10, 11,
- 11, 12, 13, 15, 4, 4, 5, 5, 5, 5, 15, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1,
- 6, 6, 6, 7, 7, 8, 8, 9, 10, 11, 12, 13, 15, 2, 2, 2, 2, 2, 2, 2, 2, 12, 13, 15, 3, 3, 3, 3, 3, 3, 3, 3,
- 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0 },
- new int[] {
- 15, 14, 15, 15, 12, 14, 15, 12, 12, 11, 12, 13, 14, 13, 13, 13, 14, 15, 15, 9, 14, 9, 15, 10, 11, 11, 12, 8, 8, 8, 8, 9,
- 9, 10, 10, 10, 11, 11, 12, 13, 13, 14, 10, 11, 11, 12, 13, 13, 14, 13, 13, 14, 15, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11,
- 10, 10, 11, 11, 12, 13, 13, 14, 15, 4, 4, 4, 4, 15, 5, 5, 5, 5, 5, 5, 5, 15, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7,
- 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 13, 14, 14, 15, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7,
- 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
- new int[] {
- 15, 15, 15, 15, 14, 14, 15, 13, 13, 13, 13, 13, 14, 14, 15, 12, 14, 15, 15, 11, 11, 12, 13, 13, 13, 14, 14, 15, 15, 10, 12, 12,
- 12, 13, 13, 13, 14, 14, 15, 15, 9, 9, 9, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 15, 15, 8, 8, 8, 8, 9, 9, 9,
- 9, 9, 9, 10, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 14, 14, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12,
- 13, 13, 14, 14, 14, 15, 15, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
- 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 4, 4, 4, 4, 4, 4, 4, 15, 15, 5, 5, 5, 5, 5,
- 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 14, 14, 14, 15, 15, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7,
- 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9,
- 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15,
- 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5,
- 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
- 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
- };
-
- for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) {
- BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits);
- assertEquals(NUM_COLUMNS, bd.getNumColumns());
- assertEquals(1 << numBucketBits, bd.getNumBuckets());
- for (int i = 0; i < bd.getNumBuckets(); ++i) {
- assertEquals(expected[numBucketBits - MIN_BUCKETBITS][i], bd.getColumn(new BucketId(16, i)));
- }
- }
- }
-
- public void testNumBucketBits() {
- Random rnd = new Random();
-
- BucketDistribution bd = new BucketDistribution(1, 4);
- for (int i = 0; i <= 0xf; ++i) {
- assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 4) & i)));
- }
-
- bd = new BucketDistribution(1, 8);
- for (int i = 0; i <= 0xff; ++i) {
- assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 8) & i)));
- }
-
- bd = new BucketDistribution(1, 16);
- for (int i = 0; i <= 0xffff; ++i) {
- assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 16) & i)));
- }
- }
-}
+package com.yahoo.vdslib; + +import com.yahoo.document.BucketId; + +import java.util.Random; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class BucketDistributionTestCase extends junit.framework.TestCase { + + private static final int NUM_COLUMNS = 16; + private static final int MIN_BUCKETBITS = 4; + private static final int MAX_BUCKETBITS = 9; + + public void printExpected() { + System.out.println(" int[][] expected = new int[][] {"); + for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) { + System.out.print(" new int[] {"); + BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits); + for (int i = 0; i < bd.getNumBuckets(); ++i) { + if (i % 32 == 0) { + System.out.println(""); + System.out.print(" "); + } + System.out.print(bd.getColumn(new BucketId(16, i))); + if (i < bd.getNumBuckets() - 1) { + System.out.print(", "); + } + } + System.out.print(" }"); + if (numBucketBits < MAX_BUCKETBITS) { + System.out.print(","); + } + System.out.println(""); + } + System.out.println(" };"); + } + + public void testDistribution() { + int[][] expected = new int[][] { + new int[] { + 10, 11, 9, 6, 4, 8, 14, 1, 13, 2, 12, 3, 5, 7, 15, 0 }, + new int[] { + 11, 12, 11, 13, 8, 13, 8, 9, 4, 5, 6, 12, 10, 15, 1, 1, 7, 9, 14, 2, 2, 14, 3, 3, 4, 5, 6, 7, 10, 15, 0, 0 }, + new int[] { + 13, 13, 13, 13, 9, 11, 8, 9, 11, 14, 9, 11, 14, 14, 8, 10, 11, 14, 4, 5, 5, 6, 6, 7, 8, 10, 12, 15, 1, 1, 1, 1, + 6, 7, 8, 10, 12, 15, 2, 2, 2, 2, 12, 15, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6, 7, 7, 9, 10, 12, 15, 0, 0, 0, 0 }, + new int[] { + 14, 14, 14, 13, 11, 14, 14, 10, 13, 14, 10, 12, 8, 8, 9, 10, 9, 10, 11, 12, 13, 15, 11, 12, 13, 13, 15, 8, 8, 9, 10, 11, + 11, 12, 13, 15, 4, 4, 5, 5, 5, 5, 15, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1, + 6, 6, 6, 7, 7, 8, 8, 9, 10, 11, 12, 13, 15, 2, 2, 2, 2, 2, 2, 2, 2, 12, 13, 15, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9, 10, 11, 12, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0 }, + new int[] { + 15, 14, 15, 15, 12, 14, 15, 12, 12, 11, 12, 13, 14, 13, 13, 13, 14, 15, 15, 9, 14, 9, 15, 10, 11, 11, 12, 8, 8, 8, 8, 9, + 9, 10, 10, 10, 11, 11, 12, 13, 13, 14, 10, 11, 11, 12, 13, 13, 14, 13, 13, 14, 15, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, 11, + 10, 10, 11, 11, 12, 13, 13, 14, 15, 4, 4, 4, 4, 15, 5, 5, 5, 5, 5, 5, 5, 15, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 14, 15, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 13, 14, 14, 15, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, + 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 12, 12, 13, 14, 15, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + new int[] { + 15, 15, 15, 15, 14, 14, 15, 13, 13, 13, 13, 13, 14, 14, 15, 12, 14, 15, 15, 11, 11, 12, 13, 13, 13, 14, 14, 15, 15, 10, 12, 12, + 12, 13, 13, 13, 14, 14, 15, 15, 9, 9, 9, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 15, 15, 8, 8, 8, 8, 9, 9, 9, + 9, 9, 9, 10, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 14, 14, 14, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12, + 13, 13, 14, 14, 14, 15, 15, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, + 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 4, 4, 4, 4, 4, 4, 4, 15, 15, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 14, 14, 14, 15, 15, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, + 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 15, 15, 15, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } + }; + + for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) { + BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits); + assertEquals(NUM_COLUMNS, bd.getNumColumns()); + assertEquals(1 << numBucketBits, bd.getNumBuckets()); + for (int i = 0; i < bd.getNumBuckets(); ++i) { + assertEquals(expected[numBucketBits - MIN_BUCKETBITS][i], bd.getColumn(new BucketId(16, i))); + } + } + } + + public void testNumBucketBits() { + Random rnd = new Random(); + + BucketDistribution bd = new BucketDistribution(1, 4); + for (int i = 0; i <= 0xf; ++i) { + assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 4) & i))); + } + + bd = new BucketDistribution(1, 8); + for (int i = 0; i <= 0xff; ++i) { + assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 8) & i))); + } + + bd = new BucketDistribution(1, 16); + for (int i = 0; i <= 0xffff; ++i) { + assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 16) & i))); + } + } +} diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index c685a9b884f..e490118370a 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -172,4 +172,4 @@ template uint64_t vdslib::Parameters::get(const vespalib::stringref &, uint64_t) template double vdslib::Parameters::get(const vespalib::stringref &, double) const; template std::string vdslib::Parameters::get(const vespalib::stringref &, std::string) const; -VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vdslib::Parameters::Value);
\ No newline at end of file +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vdslib::Parameters::Value); diff --git a/vespa-hadoop/src/test/pig/feed_operations.pig b/vespa-hadoop/src/test/pig/feed_operations.pig index 327181d4410..3939d43d68e 100644 --- a/vespa-hadoop/src/test/pig/feed_operations.pig +++ b/vespa-hadoop/src/test/pig/feed_operations.pig @@ -7,4 +7,4 @@ DEFINE VespaStorage com.yahoo.vespa.hadoop.pig.VespaStorage(); metrics = LOAD 'src/test/resources/operations_data.json' AS (data:chararray); -- Store into Vespa -STORE metrics INTO '$ENDPOINT' USING VespaStorage();
\ No newline at end of file +STORE metrics INTO '$ENDPOINT' USING VespaStorage(); diff --git a/vespa-hadoop/src/test/pig/feed_operations_with_json_loader.pig b/vespa-hadoop/src/test/pig/feed_operations_with_json_loader.pig index 6d31201e4eb..e6163591997 100644 --- a/vespa-hadoop/src/test/pig/feed_operations_with_json_loader.pig +++ b/vespa-hadoop/src/test/pig/feed_operations_with_json_loader.pig @@ -10,4 +10,4 @@ DEFINE VespaStorage com.yahoo.vespa.hadoop.pig.VespaStorage(); metrics = LOAD 'src/test/resources/operations_data.json' USING VespaJsonLoader() AS (data:chararray); -- Store into Vespa -STORE metrics INTO '$ENDPOINT' USING VespaStorage();
\ No newline at end of file +STORE metrics INTO '$ENDPOINT' USING VespaStorage(); diff --git a/vespa-hadoop/src/test/pig/feed_operations_xml.pig b/vespa-hadoop/src/test/pig/feed_operations_xml.pig index d109d56ad1e..3b06753a045 100644 --- a/vespa-hadoop/src/test/pig/feed_operations_xml.pig +++ b/vespa-hadoop/src/test/pig/feed_operations_xml.pig @@ -7,4 +7,4 @@ DEFINE VespaStorage com.yahoo.vespa.hadoop.pig.VespaStorage(); data = LOAD 'src/test/resources/operations_data.xml' AS (data:chararray); -- Store into Vespa -STORE data INTO '$ENDPOINT' USING VespaStorage();
\ No newline at end of file +STORE data INTO '$ENDPOINT' USING VespaStorage(); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java index c9997780f56..09f0185ca3e 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java @@ -32,4 +32,4 @@ public enum ErrorCode { public boolean isTransient() { return _transient; } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java index b03b6f6340c..f622d9cb9a4 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java @@ -78,4 +78,4 @@ class ByteBufferInputStream extends InputStream { } return size; } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java index b25678d2b9d..8b6c46fb1cd 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java @@ -68,4 +68,4 @@ class DocumentSendInfo { Document getDocument() { return document; } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java index e2145497589..32ed88561c9 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java @@ -81,4 +81,4 @@ public class SimpleLoggerResultCallbackTest { assertThat(outputList.toString(), containsString("Failure: fooError")); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java index 598bf6d7a4b..8dafe173b22 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java @@ -35,4 +35,4 @@ public class DocumentTest { Document document = new Document("id", "data", null /* context */); document.getData().compact(); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java index c1ca6f44c04..8265c31b21f 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java @@ -73,4 +73,4 @@ public class ThrottlePolicyTest { max = 130; assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(116)); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java index b263580df9c..965cb412512 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java @@ -256,4 +256,4 @@ public class XmlFeedReaderTest { @Test public void testAposData() throws Exception { verifyNoTransformationOfXml(feedResource4); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java index 79d66be9f97..61ff064e9a9 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java @@ -28,4 +28,4 @@ public class FeedClientImplTest { public void testCloseWaitZeroOperations() { assertThat(FeedClientImpl.waitForOperations(Instant.now(), 0, sleepValueMillis, 2000), is(false)); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java index d1ef6c0d6af..3925ee382bf 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java @@ -53,4 +53,4 @@ public class GatewayThrottlerTest { } assertTrue(x < 5000); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java index 8c90c1d4f92..60a737c490b 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java @@ -124,4 +124,4 @@ public class IOThreadTest { assert (latch.await(120, TimeUnit.SECONDS)); } } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java index 12d258b2478..99df48e2c70 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java @@ -60,4 +60,4 @@ public class ConcurrentDocumentOperationBlockerTest { blocker.operationDone(); assertTrue(latch.await(120, TimeUnit.SECONDS)); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java index f63971262e0..4452d4a5ce3 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java @@ -266,4 +266,4 @@ public class IncompleteResultsThrottlerTest { int distance = Math.abs(sweetSpot - size); return 1 + 20 * distance; } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java index 0075e89a2f3..c1953c18c2a 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java @@ -138,4 +138,4 @@ public class CommandLineArgumentsTest { assertThat(hosts, hasItem("hostValue2")); assertThat(hosts, hasItem("hostValue3")); } -}
\ No newline at end of file +} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java index eba8791bbb7..0e6510ceaa3 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java @@ -260,4 +260,4 @@ public class JsonReaderTest { assertThat(session.documentIds.size(), is(1)); assertThat(session.documentIds.get(0), is("id:foo:music:doc:foo:bar")); } -}
\ No newline at end of file +} diff --git a/vespabase/src/vespa-configserver.service.in b/vespabase/src/vespa-configserver.service.in index 8928fe87f20..8941be4f216 100644 --- a/vespabase/src/vespa-configserver.service.in +++ b/vespabase/src/vespa-configserver.service.in @@ -8,4 +8,4 @@ ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-configserver ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-configserver [Install] -WantedBy=multi-user.target
\ No newline at end of file +WantedBy=multi-user.target diff --git a/vespabase/src/vespa.service.in b/vespabase/src/vespa.service.in index f46c7ecdcb6..c6d25d39296 100644 --- a/vespabase/src/vespa.service.in +++ b/vespabase/src/vespa.service.in @@ -8,4 +8,4 @@ ExecStart=@CMAKE_INSTALL_PREFIX@/bin/vespa-start-services ExecStop=@CMAKE_INSTALL_PREFIX@/bin/vespa-stop-services [Install] -WantedBy=multi-user.target
\ No newline at end of file +WantedBy=multi-user.target diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java index cbc816b4e09..61130be18b2 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/Response.java @@ -47,4 +47,4 @@ public class Response extends HttpResponse { @Override public String getContentType() { return "application/json"; } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java index a4e0ddd1748..9fcd0d44aed 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java @@ -30,4 +30,4 @@ public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler { } } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java index aadfe5852ce..9c083af18cf 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java @@ -23,4 +23,4 @@ public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler { return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespaget"))); } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java index 0e00e8b2685..abb238c1143 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java @@ -1,2 +1,2 @@ // TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7. -package com.yahoo.feedhandler;
\ No newline at end of file +package com.yahoo.feedhandler; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 8c1344efca1..889a06d1085 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -355,4 +355,4 @@ class ClientFeederV3 { } } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java index 240b02fabc7..97c825c005e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java @@ -37,4 +37,4 @@ public class ClientState { this.operationsForOpsPerSec = operationsForOpsPerSec; } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 84a46484cfe..aecf4a43c71 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -67,4 +67,4 @@ public class FeedReplyReader implements ReplyHandler { Thread.currentThread().interrupt(); } } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java index 64bc687d425..3089f05f394 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java @@ -72,4 +72,4 @@ public class FeederSettings { } } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java index 087cb11c350..beae9b0634b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java @@ -24,4 +24,4 @@ public final class MetricNames { private MetricNames() { } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java index 39392fdefc4..d46bce1a789 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java @@ -24,4 +24,4 @@ public class ReplyContext { this.creationTime = System.currentTimeMillis(); } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java index ae96a25fad3..34c62ab1fec 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java @@ -36,4 +36,4 @@ public final class MetaStream extends ByteArrayInputStream { return operations[i++]; } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java index 1208ce334d9..c299568535b 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java @@ -66,4 +66,4 @@ class MockNetwork implements Network { return null; } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java index 298c925d032..3150665cd71 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java @@ -32,4 +32,4 @@ class MockReply extends Reply { return context; } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java index 915a8635863..67e377b33a1 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java @@ -159,4 +159,4 @@ public class V3CongestionTestCase { clientFeederV3.handleRequest(request); assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES); } -}
\ No newline at end of file +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java index a4ac0f4fdaf..93f606e72a9 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java @@ -72,4 +72,4 @@ public class MockReader implements FeedReader { } } -}
\ No newline at end of file +} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java index 52f5add9f44..10ef5447f06 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SimpleFeedAccess.java @@ -16,4 +16,4 @@ public interface SimpleFeedAccess { void update(DocumentUpdate update, TestAndSetCondition condition); boolean isAborted(); -}
\ No newline at end of file +} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java index 4e0896b6da4..893f83a1080 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java @@ -35,4 +35,4 @@ public final class MetricResponse extends HttpResponse { return "application/xml"; } -}
\ No newline at end of file +} diff --git a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterDef.java b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterDef.java index ab404893c08..ced90ce4aeb 100644 --- a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterDef.java +++ b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterDef.java @@ -12,4 +12,4 @@ public class ClusterDef { public String getName() { return name; } public String getConfigId() { return configId; } -}
\ No newline at end of file +} diff --git a/vespaclient-java/src/test/files/myfeed.xml b/vespaclient-java/src/test/files/myfeed.xml index 008d4fcba13..f1e6ddb8966 100644 --- a/vespaclient-java/src/test/files/myfeed.xml +++ b/vespaclient-java/src/test/files/myfeed.xml @@ -2,4 +2,4 @@ <document documentid="doc:test:foo" documenttype="news"/> <update documentid="doc:test:foo" documenttype="news"/> <remove documentid="doc:test:foo"/> -</vespafeed>
\ No newline at end of file +</vespafeed> diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java b/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java index 389fe8a85ea..aeab747242a 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java @@ -1,140 +1,140 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.concurrent;
-
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Reference implementation of the 'Incremental Minimal Event Barrier'
- * algorithm. An event in this context is defined to be something that
- * happens during a time interval. An event barrier is a time interval
- * for which events may start before or end after, but not both. The
- * problem solved by the algorithm is to determine the minimal event
- * barrier starting at a given time. In other words; wait for the
- * currently active events to complete. The most natural use of this
- * algorithm would be to make a thread wait for events happening in
- * other threads to complete.
- *
- * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a>
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class EventBarrier {
-
- private final List<Entry> queue = new LinkedList<>();
- private int barrierToken = 0;
- private int eventCount = 0;
-
- /**
- * At creation there are no active events and no pending barriers.
- */
- public EventBarrier() {
- // empty
- }
-
- /**
- * Obtain the current number of active events. This method is
- * intended for testing and debugging.
- *
- * @return Number of active events.
- */
- int getNumEvents() {
- int cnt = eventCount;
- for (Entry entry : queue) {
- cnt += entry.eventCount;
- }
- return cnt;
- }
-
- /**
- * Obtain the current number of pending barriers. This method is
- * intended for testing and debugging.
- *
- * @return Number of pending barriers.
- */
- int getNumBarriers() {
- return queue.size();
- }
-
- /**
- * Signal the start of an event. The value returned from this
- * method must later be passed to the completeEvent method when
- * signaling the completion of the event.
- *
- * @return Opaque token identifying the started event.
- */
- public int startEvent() {
- ++eventCount;
- return barrierToken;
- }
-
- /**
- * Signal the completion of an event. The value passed to this
- * method must be the same as the return value previously obtained
- * from the startEvent method. This method will signal the
- * completion of all pending barriers that were completed by the
- * completion of this event.
- *
- * @param token Opaque token identifying the completed event.
- */
- public void completeEvent(int token) {
- if (token == this.barrierToken) {
- --eventCount;
- return;
- }
- --queue.get(queue.size() - (this.barrierToken - token)).eventCount;
- while (!queue.isEmpty() && queue.get(0).eventCount == 0) {
- queue.remove(0).handler.completeBarrier();
- }
- }
-
- /**
- * Initiate the detection of the minimal event barrier starting
- * now. If this method returns false it means that no events were
- * currently active and the minimal event barrier was infinitely
- * small. If this method returns false the handler will not be
- * notified of the completion of the barrier. If this method
- * returns true it means that the started barrier is pending and
- * that the handler passed to this method will be notified of its
- * completion at a later time.
- *
- * @param handler Handler notified of the completion of the barrier.
- * @return True if a barrier was started, false if no events were active.
- */
- public boolean startBarrier(BarrierWaiter handler) {
- if (eventCount == 0 && queue.isEmpty()) {
- return false;
- }
- queue.add(new Entry(eventCount, handler));
- ++barrierToken;
- eventCount = 0;
- return true;
- }
-
- /**
- * Declares the interface required to wait for the detection of a
- * minimal event barrier. An object that implements this is passed
- * to the {@link EventBarrier#startBarrier(BarrierWaiter)}.
- */
- public interface BarrierWaiter {
-
- /**
- * Callback invoked by the thread that detected the minimal
- * event barrier. Once this is called, all events taking place
- * at or before the corresponding call to {@link
- * EventBarrier#startBarrier(BarrierWaiter)} have ended.
- */
- public void completeBarrier();
- }
-
- private static class Entry {
-
- int eventCount;
- final BarrierWaiter handler;
-
- Entry(int eventCount, BarrierWaiter handler) {
- this.eventCount = eventCount;
- this.handler = handler;
- }
- }
-}
+package com.yahoo.concurrent; + + +import java.util.LinkedList; +import java.util.List; + +/** + * Reference implementation of the 'Incremental Minimal Event Barrier' + * algorithm. An event in this context is defined to be something that + * happens during a time interval. An event barrier is a time interval + * for which events may start before or end after, but not both. The + * problem solved by the algorithm is to determine the minimal event + * barrier starting at a given time. In other words; wait for the + * currently active events to complete. The most natural use of this + * algorithm would be to make a thread wait for events happening in + * other threads to complete. + * + * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class EventBarrier { + + private final List<Entry> queue = new LinkedList<>(); + private int barrierToken = 0; + private int eventCount = 0; + + /** + * At creation there are no active events and no pending barriers. + */ + public EventBarrier() { + // empty + } + + /** + * Obtain the current number of active events. This method is + * intended for testing and debugging. + * + * @return Number of active events. + */ + int getNumEvents() { + int cnt = eventCount; + for (Entry entry : queue) { + cnt += entry.eventCount; + } + return cnt; + } + + /** + * Obtain the current number of pending barriers. This method is + * intended for testing and debugging. + * + * @return Number of pending barriers. + */ + int getNumBarriers() { + return queue.size(); + } + + /** + * Signal the start of an event. The value returned from this + * method must later be passed to the completeEvent method when + * signaling the completion of the event. + * + * @return Opaque token identifying the started event. + */ + public int startEvent() { + ++eventCount; + return barrierToken; + } + + /** + * Signal the completion of an event. The value passed to this + * method must be the same as the return value previously obtained + * from the startEvent method. This method will signal the + * completion of all pending barriers that were completed by the + * completion of this event. + * + * @param token Opaque token identifying the completed event. + */ + public void completeEvent(int token) { + if (token == this.barrierToken) { + --eventCount; + return; + } + --queue.get(queue.size() - (this.barrierToken - token)).eventCount; + while (!queue.isEmpty() && queue.get(0).eventCount == 0) { + queue.remove(0).handler.completeBarrier(); + } + } + + /** + * Initiate the detection of the minimal event barrier starting + * now. If this method returns false it means that no events were + * currently active and the minimal event barrier was infinitely + * small. If this method returns false the handler will not be + * notified of the completion of the barrier. If this method + * returns true it means that the started barrier is pending and + * that the handler passed to this method will be notified of its + * completion at a later time. + * + * @param handler Handler notified of the completion of the barrier. + * @return True if a barrier was started, false if no events were active. + */ + public boolean startBarrier(BarrierWaiter handler) { + if (eventCount == 0 && queue.isEmpty()) { + return false; + } + queue.add(new Entry(eventCount, handler)); + ++barrierToken; + eventCount = 0; + return true; + } + + /** + * Declares the interface required to wait for the detection of a + * minimal event barrier. An object that implements this is passed + * to the {@link EventBarrier#startBarrier(BarrierWaiter)}. + */ + public interface BarrierWaiter { + + /** + * Callback invoked by the thread that detected the minimal + * event barrier. Once this is called, all events taking place + * at or before the corresponding call to {@link + * EventBarrier#startBarrier(BarrierWaiter)} have ended. + */ + public void completeBarrier(); + } + + private static class Entry { + + int eventCount; + final BarrierWaiter handler; + + Entry(int eventCount, BarrierWaiter handler) { + this.eventCount = eventCount; + this.handler = handler; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java index 5aa4990a86a..7fcb2a80265 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java @@ -1,41 +1,41 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.concurrent;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * This is an implementation of {@link Timer} that is backed by an actual system timer.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public enum SystemTimer implements Timer {
-
- INSTANCE;
-
- private volatile long millis;
-
- private SystemTimer() {
- millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
- Thread thread = new Thread() {
-
- @Override
- public void run() {
- while (true) {
- millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
- try {
- Thread.sleep(1);
- } catch (InterruptedException e) {
- break;
- }
- }
- }
- };
- thread.setDaemon(true);
- thread.start();
- }
-
- @Override
- public long milliTime() {
- return millis;
- }
-}
+package com.yahoo.concurrent; + +import java.util.concurrent.TimeUnit; + +/** + * This is an implementation of {@link Timer} that is backed by an actual system timer. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public enum SystemTimer implements Timer { + + INSTANCE; + + private volatile long millis; + + private SystemTimer() { + millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + Thread thread = new Thread() { + + @Override + public void run() { + while (true) { + millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + break; + } + } + } + }; + thread.setDaemon(true); + thread.start(); + } + + @Override + public long milliTime() { + return millis; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java index aefbfafb7b1..35d622613fd 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java @@ -1,19 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.concurrent;
-
-/**
- * This interface wraps access to some timer that can be used to measure elapsed time, in milliseconds. This
- * abstraction allows for unit testing the behavior of time-based constructs.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public interface Timer {
-
- /**
- * Returns the current value of some arbitrary timer, in milliseconds. This method can only be used to measure
- * elapsed time and is not related to any other notion of system or wall-clock time.
- *
- * @return The current value of the timer, in milliseconds.
- */
- public long milliTime();
-}
+package com.yahoo.concurrent; + +/** + * This interface wraps access to some timer that can be used to measure elapsed time, in milliseconds. This + * abstraction allows for unit testing the behavior of time-based constructs. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface Timer { + + /** + * Returns the current value of some arbitrary timer, in milliseconds. This method can only be used to measure + * elapsed time and is not related to any other notion of system or wall-clock time. + * + * @return The current value of the timer, in milliseconds. + */ + public long milliTime(); +} diff --git a/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java index a769aa4bda4..45d58014c0e 100644 --- a/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java @@ -126,4 +126,4 @@ public abstract class AbstractByteWriter extends GenericWriter implements public long appended() { return buffer.appended(); } -}
\ No newline at end of file +} diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java index 7b93429acdb..c9c6c356640 100644 --- a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java +++ b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java @@ -68,4 +68,4 @@ public class NamedReader extends Reader { } } -}
\ No newline at end of file +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java index 79e9d49c9f5..78c6e7d0a87 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java +++ b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java @@ -62,4 +62,4 @@ public final class ClassValidator { return unmaskedMethods(testClass, testClass.getSuperclass()); } -}
\ No newline at end of file +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/package-info.java b/vespajlib/src/main/java/com/yahoo/transaction/package-info.java index 72ac10d13d0..4e71a1130e8 100644 --- a/vespajlib/src/main/java/com/yahoo/transaction/package-info.java +++ b/vespajlib/src/main/java/com/yahoo/transaction/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.transaction; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java b/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java index eae792effd4..e8e27780860 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java @@ -1,168 +1,168 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.concurrent;
-
-import junit.framework.TestCase;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class EventBarrierTestCase extends TestCase {
-
- public void testEmpty() {
- // waiting for an empty set of events
- Barrier b = new Barrier();
- EventBarrier eb = new EventBarrier();
-
- assertTrue(!eb.startBarrier(b));
- assertTrue(!b.done);
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
-
- int token = eb.startEvent();
- eb.completeEvent(token);
-
- assertTrue(!eb.startBarrier(b));
- assertTrue(!b.done);
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
- }
-
- public void testSimple() {
- // a single barrier waiting for a single event
- Barrier b = new Barrier();
- EventBarrier eb = new EventBarrier();
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
-
- int token = eb.startEvent();
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 0);
-
- assertTrue(eb.startBarrier(b));
- assertTrue(!b.done);
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 1);
-
- eb.completeEvent(token);
- assertTrue(b.done);
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
- }
-
- public void testBarrierChain() {
- // more than one barrier waiting for the same set of events
- Barrier b1 = new Barrier();
- Barrier b2 = new Barrier();
- Barrier b3 = new Barrier();
- EventBarrier eb = new EventBarrier();
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
-
- int token = eb.startEvent();
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 0);
-
- assertTrue(eb.startBarrier(b1));
- assertTrue(eb.startBarrier(b2));
- assertTrue(eb.startBarrier(b3));
- assertTrue(!b1.done);
- assertTrue(!b2.done);
- assertTrue(!b3.done);
-
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 3);
-
- eb.completeEvent(token);
- assertTrue(b1.done);
- assertTrue(b2.done);
- assertTrue(b3.done);
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
- }
-
- public void testEventAfter() {
- // new events starting after the start of a barrier
- Barrier b = new Barrier();
- EventBarrier eb = new EventBarrier();
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
-
- int token = eb.startEvent();
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 0);
-
- assertTrue(eb.startBarrier(b));
- assertTrue(!b.done);
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 1);
-
- int t2 = eb.startEvent();
- assertTrue(!b.done);
- assertEquals(eb.getNumEvents(), 2);
- assertEquals(eb.getNumBarriers(), 1);
-
- eb.completeEvent(token);
- assertTrue(b.done);
- assertEquals(eb.getNumEvents(), 1);
- assertEquals(eb.getNumBarriers(), 0);
-
- eb.completeEvent(t2);
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
- }
-
- public void testReorder() {
- // events completing in a different order than they started
- Barrier b1 = new Barrier();
- Barrier b2 = new Barrier();
- Barrier b3 = new Barrier();
- EventBarrier eb = new EventBarrier();
-
- int t1 = eb.startEvent();
- eb.startBarrier(b1);
- int t2 = eb.startEvent();
- eb.startBarrier(b2);
- int t3 = eb.startEvent();
- eb.startBarrier(b3);
- int t4 = eb.startEvent();
-
- assertEquals(eb.getNumEvents(), 4);
- assertEquals(eb.getNumBarriers(), 3);
-
- assertTrue(!b1.done);
- assertTrue(!b2.done);
- assertTrue(!b3.done);
-
- eb.completeEvent(t4);
- assertTrue(!b1.done);
- assertTrue(!b2.done);
- assertTrue(!b3.done);
-
- eb.completeEvent(t3);
- assertTrue(!b1.done);
- assertTrue(!b2.done);
- assertTrue(!b3.done);
-
- eb.completeEvent(t1);
- assertTrue(b1.done);
- assertTrue(!b2.done);
- assertTrue(!b3.done);
-
- eb.completeEvent(t2);
- assertTrue(b1.done);
- assertTrue(b2.done);
- assertTrue(b3.done);
-
- assertEquals(eb.getNumEvents(), 0);
- assertEquals(eb.getNumBarriers(), 0);
- }
-
- private static class Barrier implements EventBarrier.BarrierWaiter {
- boolean done = false;
-
- @Override
- public void completeBarrier() {
- done = true;
- }
- }
-}
+package com.yahoo.concurrent; + +import junit.framework.TestCase; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class EventBarrierTestCase extends TestCase { + + public void testEmpty() { + // waiting for an empty set of events + Barrier b = new Barrier(); + EventBarrier eb = new EventBarrier(); + + assertTrue(!eb.startBarrier(b)); + assertTrue(!b.done); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + + int token = eb.startEvent(); + eb.completeEvent(token); + + assertTrue(!eb.startBarrier(b)); + assertTrue(!b.done); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + } + + public void testSimple() { + // a single barrier waiting for a single event + Barrier b = new Barrier(); + EventBarrier eb = new EventBarrier(); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + + int token = eb.startEvent(); + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 0); + + assertTrue(eb.startBarrier(b)); + assertTrue(!b.done); + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 1); + + eb.completeEvent(token); + assertTrue(b.done); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + } + + public void testBarrierChain() { + // more than one barrier waiting for the same set of events + Barrier b1 = new Barrier(); + Barrier b2 = new Barrier(); + Barrier b3 = new Barrier(); + EventBarrier eb = new EventBarrier(); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + + int token = eb.startEvent(); + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 0); + + assertTrue(eb.startBarrier(b1)); + assertTrue(eb.startBarrier(b2)); + assertTrue(eb.startBarrier(b3)); + assertTrue(!b1.done); + assertTrue(!b2.done); + assertTrue(!b3.done); + + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 3); + + eb.completeEvent(token); + assertTrue(b1.done); + assertTrue(b2.done); + assertTrue(b3.done); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + } + + public void testEventAfter() { + // new events starting after the start of a barrier + Barrier b = new Barrier(); + EventBarrier eb = new EventBarrier(); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + + int token = eb.startEvent(); + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 0); + + assertTrue(eb.startBarrier(b)); + assertTrue(!b.done); + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 1); + + int t2 = eb.startEvent(); + assertTrue(!b.done); + assertEquals(eb.getNumEvents(), 2); + assertEquals(eb.getNumBarriers(), 1); + + eb.completeEvent(token); + assertTrue(b.done); + assertEquals(eb.getNumEvents(), 1); + assertEquals(eb.getNumBarriers(), 0); + + eb.completeEvent(t2); + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + } + + public void testReorder() { + // events completing in a different order than they started + Barrier b1 = new Barrier(); + Barrier b2 = new Barrier(); + Barrier b3 = new Barrier(); + EventBarrier eb = new EventBarrier(); + + int t1 = eb.startEvent(); + eb.startBarrier(b1); + int t2 = eb.startEvent(); + eb.startBarrier(b2); + int t3 = eb.startEvent(); + eb.startBarrier(b3); + int t4 = eb.startEvent(); + + assertEquals(eb.getNumEvents(), 4); + assertEquals(eb.getNumBarriers(), 3); + + assertTrue(!b1.done); + assertTrue(!b2.done); + assertTrue(!b3.done); + + eb.completeEvent(t4); + assertTrue(!b1.done); + assertTrue(!b2.done); + assertTrue(!b3.done); + + eb.completeEvent(t3); + assertTrue(!b1.done); + assertTrue(!b2.done); + assertTrue(!b3.done); + + eb.completeEvent(t1); + assertTrue(b1.done); + assertTrue(!b2.done); + assertTrue(!b3.done); + + eb.completeEvent(t2); + assertTrue(b1.done); + assertTrue(b2.done); + assertTrue(b3.done); + + assertEquals(eb.getNumEvents(), 0); + assertEquals(eb.getNumBarriers(), 0); + } + + private static class Barrier implements EventBarrier.BarrierWaiter { + boolean done = false; + + @Override + public void completeBarrier() { + done = true; + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java index ace911409bc..a4077592fc7 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java @@ -67,4 +67,4 @@ public class ConcatTestCase { assertEquals(Tensor.from("tensor(x[2],y[4]):{{x:0,y:0}:1.0,{x:0,y:1}:3.0,{x:0,y:2}:5.0,{x:0,y:3}:6.0,{x:1,y:0}:2.0,{x:1,y:1}:4.0,{x:1,y:2}:5.0,{x:1,y:3}:6.0}"), a.concat(b, "y")); } -}
\ No newline at end of file +} diff --git a/vsm/src/vespa/vsm/common/document.cpp b/vsm/src/vespa/vsm/common/document.cpp index 94aa1a0909b..d45b7d4f1e4 100644 --- a/vsm/src/vespa/vsm/common/document.cpp +++ b/vsm/src/vespa/vsm/common/document.cpp @@ -70,4 +70,4 @@ Document::~Document() { } } VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vsm::FieldIdTList); -VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vsm::IndexFieldMapT);
\ No newline at end of file +VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vsm::IndexFieldMapT); diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java b/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java index b92df20ab8c..aba92b986de 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java @@ -21,4 +21,4 @@ public class ChainCycleException extends RuntimeException { public List<?> components() { return components; } -}
\ No newline at end of file +} diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/package-info.java index 95a1a013f3c..bcbc551ac8d 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/package-info.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/recipes/package-info.java @@ -2,4 +2,4 @@ @ExportPackage package com.yahoo.vespa.curator.recipes; -import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/package-info.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/package-info.java index 9977c6aa062..5c4745e13a7 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/package-info.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/transaction/package-info.java @@ -6,4 +6,4 @@ import com.yahoo.osgi.annotation.ExportPackage; /** * Classes which allows curator operations to participate in two-phase transactions over multiple systems. - */
\ No newline at end of file + */ |