diff options
72 files changed, 532 insertions, 2182 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index bad32f87a8c..9089da68e10 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -1,25 +1,35 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class Communicator - * - * Responsible for doing RPC requests to VDS nodes. - */ - package com.yahoo.vespa.clustercontroller.core.rpc; -import com.yahoo.jrt.*; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int8Value; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.Values; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.State; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.clustercontroller.core.*; +import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; +import com.yahoo.vespa.clustercontroller.core.Communicator; +import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions; +import com.yahoo.vespa.clustercontroller.core.GetNodeStateRequest; +import com.yahoo.vespa.clustercontroller.core.NodeInfo; +import com.yahoo.vespa.clustercontroller.core.SetClusterStateRequest; +import com.yahoo.vespa.clustercontroller.core.Timer; import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkArgument; /** - * This class is not thread-safe. + * Responsible for doing RPC requests to VDS nodes. + This class is not thread-safe. */ public class RPCCommunicator implements Communicator { @@ -43,14 +53,13 @@ public class RPCCommunicator implements Communicator { return new Supervisor(new Transport()); } - public RPCCommunicator( - final Supervisor supervisor, - final Timer t, - final int index, - final int nodeStateRequestTimeoutIntervalMaxMs, - final int nodeStateRequestTimeoutIntervalStartPercentage, - final int nodeStateRequestTimeoutIntervalStopPercentage, - final int nodeStateRequestRoundTripTimeMaxSeconds) { + public RPCCommunicator(Supervisor supervisor, + Timer t, + int index, + int nodeStateRequestTimeoutIntervalMaxMs, + int nodeStateRequestTimeoutIntervalStartPercentage, + int nodeStateRequestTimeoutIntervalStopPercentage, + int nodeStateRequestRoundTripTimeMaxSeconds) { this.timer = t; this.fleetControllerIndex = index; checkArgument(nodeStateRequestTimeoutIntervalMaxMs > 0); @@ -73,14 +82,13 @@ public class RPCCommunicator implements Communicator { public Target getConnection(final NodeInfo node) { Target t = node.getConnection(); if (t == null || !t.isValid()) { - t = node.setConnection( - supervisor.connect(new Spec(node.getRpcAddress()))); + t = node.setConnection(supervisor.connect(new Spec(node.getRpcAddress()))); } return t; } @Override - public void propagateOptions(final FleetControllerOptions options) { + public void propagateOptions(FleetControllerOptions options) { checkArgument(options.nodeStateRequestTimeoutMS > 0); checkArgument(options.nodeStateRequestTimeoutEarliestPercentage >= 0); checkArgument(options.nodeStateRequestTimeoutEarliestPercentage <= 100); @@ -121,15 +129,15 @@ public class RPCCommunicator implements Communicator { @Override public void setSystemState(ClusterStateBundle stateBundle, NodeInfo node, Waiter<SetClusterStateRequest> externalWaiter) { - final RPCSetClusterStateWaiter waiter = new RPCSetClusterStateWaiter(externalWaiter, timer); - final ClusterState baselineState = stateBundle.getBaselineClusterState(); + RPCSetClusterStateWaiter waiter = new RPCSetClusterStateWaiter(externalWaiter, timer); + ClusterState baselineState = stateBundle.getBaselineClusterState(); Target connection = getConnection(node); - if (!connection.isValid()) { + if ( ! connection.isValid()) { log.log(LogLevel.DEBUG, "Connection to " + node.getRpcAddress() + " could not be created."); return; } - final int nodeVersion = node.getVersion(); + int nodeVersion = node.getVersion(); Request req; if (nodeVersion <= 2) { req = new Request(LEGACY_SET_SYSTEM_STATE2_RPC_METHOD_NAME); @@ -155,13 +163,13 @@ public class RPCCommunicator implements Communicator { // protected for testing. protected int generateNodeStateRequestTimeoutMs() { - final double intervalFraction = Math.random(); - final double earliestTimeoutSeconds = + double intervalFraction = Math.random(); + double earliestTimeoutSeconds = nodeStateRequestTimeoutIntervalMaxSeconds * nodeStateRequestTimeoutIntervalStartPercentage / 100.0; - final double latestTimeoutSeconds = + double latestTimeoutSeconds = nodeStateRequestTimeoutIntervalMaxSeconds * nodeStateRequestTimeoutIntervalStopPercentage / 100.0; - final double interval = latestTimeoutSeconds - earliestTimeoutSeconds; - final double timeoutSeconds = earliestTimeoutSeconds + intervalFraction * interval; + double interval = latestTimeoutSeconds - earliestTimeoutSeconds; + double timeoutSeconds = earliestTimeoutSeconds + intervalFraction * interval; return (int) (timeoutSeconds * 1000); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index ed4192941b4..52cd0a27933 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -143,11 +143,9 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (connection == null) continue; log.log(LogLevel.DEBUG, "Got a status page request."); String requestString = ""; - BufferedReader br = null; OutputStream output = null; - try{ + try (BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { StringBuilder sb = new StringBuilder(); - br = new BufferedReader(new InputStreamReader(connection.getInputStream())); while (true) { String s = br.readLine(); if (s == null) throw new java.io.IOException("No data in HTTP request on socket " + connection.toString()); @@ -235,17 +233,12 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { log.log(LogLevel.WARNING, "Caught exception in HTTP server thread: " + e.getClass().getName() + ": " + e.getMessage()); } finally { - if (output != null) try{ + if (output != null) try { output.close(); } catch (IOException e) { log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? LogLevel.DEBUG : LogLevel.INFO, "Failed to close output stream on socket " + connection + ": " + e.getMessage()); } - if (br != null) try { - br.close(); - } catch (IOException e) { - log.log(LogLevel.INFO, "Failed to close input stream on socket " + connection + ": " + e.getMessage()); - } if (connection != null) try{ connection.close(); } catch (IOException e) { diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index d3cab74c66f..c38f7aec8c6 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -34,6 +34,7 @@ import java.util.logging.Level; import java.util.logging.Logger; public class RestApiHandler implements HttpRequestHandler { + public static final Duration MAX_TIMEOUT = Duration.ofHours(1); private final static Logger log = Logger.getLogger(RestApiHandler.class.getName()); @@ -56,8 +57,7 @@ public class RestApiHandler implements HttpRequestHandler { private static void logRequestException(HttpRequest request, Exception exception, Level level) { String exceptionString = Exceptions.toMessageString(exception); - log.log(level, "Failed to process request with URI path " + - request.getPath() + ": " + exceptionString); + log.log(level, "Failed to process request with URI path " + request.getPath() + ": " + exceptionString); } @Override diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java index f891d0c0a0a..44180b3f863 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -72,13 +72,14 @@ public class CppClassBuilder implements ClassBuilder { String readFile(File f) throws IOException { if (!f.isFile()) return null; StringBuilder sb = new StringBuilder(); - BufferedReader sr = new BufferedReader(new FileReader(f)); - while (true) { - String line = sr.readLine(); - if (line == null) break; - sb.append(line).append("\n"); + try (BufferedReader sr = new BufferedReader(new FileReader(f))) { + while (true) { + String line = sr.readLine(); + if (line == null) break; + sb.append(line).append("\n"); + } + return sb.toString(); } - return sb.toString(); } void writeFile(File f, String content) throws IOException { diff --git a/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java index 849b4f34914..3250fc69ae8 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/NormalizedDefinition.java @@ -19,7 +19,8 @@ import java.nio.charset.Charset; * @author hmusum */ public class NormalizedDefinition { - /* Patterns used for finding ranges in config definitions */ + + // Patterns used for finding ranges in config definitions private static final Pattern intPattern = Pattern.compile(".*int.*range.*"); private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); private MessageDigest md5; diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 33896c870a5..1f56523bce6 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -5,6 +5,7 @@ import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.core.AccessLogConfig; import com.yahoo.io.NativeIO; import com.yahoo.log.LogFileDb; +import com.yahoo.system.ProcessExecuter; import java.io.File; import java.io.FileInputStream; @@ -25,7 +26,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; - /** * <p>Implements log file naming/rotating logic for container logs.</p> * @@ -226,7 +226,7 @@ public class LogFileHandler extends StreamHandler { } // Throw InterruptedException upwards rather than relying on isInterrupted to stop the thread as - // isInterrupted() returns false after inerruption in p.waitFor + // isInterrupted() returns false after interruption in p.waitFor private void internalRotateNow() throws InterruptedException { // figure out new file name, then // use super.setOutputStream to switch to a new file @@ -295,7 +295,7 @@ public class LogFileHandler extends StreamHandler { } /** Name files by date - create a symlink with a constant name to the newest file */ - private void createSymlinkToCurrentFile() throws InterruptedException { + private void createSymlinkToCurrentFile() { if (symlinkName == null) return; File f = new File(fileName); File f2 = new File(f.getParent(), symlinkName); @@ -308,11 +308,9 @@ public class LogFileHandler extends StreamHandler { } String [] cmd = new String[]{"/bin/ln", "-sf", canonicalPath, f2.getPath()}; try { - Runtime r = Runtime.getRuntime(); - Process p = r.exec(cmd); + int retval = new ProcessExecuter().exec(cmd).getFirst(); // Detonator pattern: Think of all the fun we can have if ln isn't what we // think it is, if it doesn't return, etc, etc - int retval = p.waitFor(); if (retval != 0) { logger.warning("Command '" + Arrays.toString(cmd) + "' + failed with exitcode=" + retval); } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java index 0cedfc0f693..adb4583d405 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFormatter.java @@ -101,7 +101,7 @@ public class LogFormatter extends Formatter { * <li>%% % * </ul> */ - public static final String insertDate (String pattern, long time) { + public static String insertDate(String pattern, long time) { DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS Z"); df.setTimeZone(TimeZone.getTimeZone("UTC")); Date date = new Date(time); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index eeb5a3af79a..08dcbe17db2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -43,7 +43,7 @@ public class FS4InvokerFactory { } public SearchInvoker getSearchInvoker(Query query, SearchCluster.Node node) { - return new FS4SearchInvoker(searcher, query, fs4ResourcePool, node.hostname(), node.fs4port(), node.key()); + return new FS4SearchInvoker(searcher, query, fs4ResourcePool, node); } public Optional<SearchInvoker> getSearchInvoker(Query query, List<SearchCluster.Node> nodes) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java index 82f87fcac19..ac48aef7063 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4SearchInvoker.java @@ -11,10 +11,13 @@ import com.yahoo.fs4.mplex.FS4Channel; import com.yahoo.fs4.mplex.InvalidChannelException; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.dispatch.SearchCluster; import com.yahoo.search.dispatch.SearchInvoker; +import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -30,18 +33,17 @@ import static java.util.Arrays.asList; public class FS4SearchInvoker extends SearchInvoker { private final VespaBackEndSearcher searcher; private FS4Channel channel; - private final Optional<Integer> distributionKey; + private final Optional<SearchCluster.Node> node; private ErrorMessage pendingSearchError = null; private Query query = null; private QueryPacket queryPacket = null; - public FS4SearchInvoker(VespaBackEndSearcher searcher, Query query, FS4ResourcePool fs4ResourcePool, String hostname, int port, - int distributionKey) { + public FS4SearchInvoker(VespaBackEndSearcher searcher, Query query, FS4ResourcePool fs4ResourcePool, SearchCluster.Node node) { this.searcher = searcher; - this.distributionKey = Optional.of(distributionKey); + this.node = Optional.of(node); - Backend backend = fs4ResourcePool.getBackend(hostname, port); + Backend backend = fs4ResourcePool.getBackend(node.hostname(), node.fs4port()); this.channel = backend.openChannel(); channel.setQuery(query); } @@ -49,7 +51,7 @@ public class FS4SearchInvoker extends SearchInvoker { // fdispatch code path public FS4SearchInvoker(VespaBackEndSearcher searcher, Query query, Backend backend) { this.searcher = searcher; - this.distributionKey = Optional.empty(); + this.node = Optional.empty(); this.channel = backend.openChannel(); channel.setQuery(query); } @@ -82,20 +84,20 @@ public class FS4SearchInvoker extends SearchInvoker { @Override protected List<Result> getSearchResults(CacheKey cacheKey) throws IOException { if(pendingSearchError != null) { - return asList(new Result(query, pendingSearchError)); + return errorResult(pendingSearchError); } BasicPacket[] basicPackets; try { basicPackets = channel.receivePackets(query.getTimeLeft(), 1); } catch (ChannelTimeoutException e) { - return asList(new Result(query, ErrorMessage.createTimeout("Timeout while waiting for " + getName()))); + return errorResult(ErrorMessage.createTimeout("Timeout while waiting for " + getName())); } catch (InvalidChannelException e) { - return asList(new Result(query, ErrorMessage.createBackendCommunicationError("Invalid channel for " + getName()))); + return errorResult(ErrorMessage.createBackendCommunicationError("Invalid channel for " + getName())); } if (basicPackets.length == 0) { - return asList(new Result(query, ErrorMessage.createBackendCommunicationError(getName() + " got no packets back"))); + return errorResult(ErrorMessage.createBackendCommunicationError(getName() + " got no packets back")); } if (isLoggingFine()) @@ -114,7 +116,7 @@ public class FS4SearchInvoker extends SearchInvoker { searcher.addMetaInfo(query, queryPacket.getQueryPacketData(), resultPacket, result); - searcher.addUnfilledHits(result, resultPacket.getDocuments(), false, queryPacket.getQueryPacketData(), cacheKey, distributionKey); + searcher.addUnfilledHits(result, resultPacket.getDocuments(), false, queryPacket.getQueryPacketData(), cacheKey, node.map(SearchCluster.Node::key)); Packet[] packets; CacheControl cacheControl = searcher.getCacheControl(); PacketWrapper packetWrapper = cacheControl.lookup(cacheKey, query); @@ -129,12 +131,21 @@ public class FS4SearchInvoker extends SearchInvoker { } else { packets = new Packet[1]; packets[0] = resultPacket; - cacheControl.cache(cacheKey, query, new DocsumPacketKey[0], packets, distributionKey); + cacheControl.cache(cacheKey, query, new DocsumPacketKey[0], packets, node.map(SearchCluster.Node::key)); } } return asList(result); } + private List<Result> errorResult(ErrorMessage errorMessage) { + Result error = new Result(query, errorMessage); + node.ifPresent(n -> { + Coverage coverage = new Coverage(0, n.getActiveDocuments(), 0); + error.setCoverage(coverage); + }); + return Arrays.asList(error); + } + @Override public void release() { if (channel != null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index b429995460d..36d283040a2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -22,6 +22,7 @@ import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.query.Ranking; +import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import edu.umd.cs.findbugs.annotations.NonNull; @@ -264,16 +265,21 @@ public class FastSearcher extends VespaBackEndSearcher { } private Result mergeResults(List<Result> results, Query query, Execution execution) { - if(results.size() == 1) { + if (results.size() == 1) { return results.get(0); } Result result = new Result(query); + // keep a separate tally of coverage as the normal merge counts using + // federated query rules + Coverage finalCoverage = new Coverage(0, 0); for (Result partialResult : results) { + finalCoverage.mergeWithPartition(partialResult.getCoverage(true)); result.mergeWith(partialResult); result.hits().addAll(partialResult.hits().asUnorderedHits()); } + result.setCoverage(finalCoverage); if (query.getOffset() != 0 || result.hits().size() > query.getHits()) { // with multiple results, each partial result is expected to have diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index cb7f5215c7c..3896e817429 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -38,7 +38,7 @@ public class ClusterMonitor<T> { private final Map<T, BaseNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); /** @deprecated use the constructor with just the first argument instead */ - @Deprecated + @Deprecated // TODO: Remove on Vespa 7 public ClusterMonitor(NodeManager<T> manager, String ignored) { this(manager); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 455696c16b1..9eac9b9b63d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.dispatch.SearchCluster.Group; @@ -24,8 +23,6 @@ public class LoadBalancer { private static final Logger log = Logger.getLogger(LoadBalancer.class.getName()); - private static final CompoundName QUERY_NODE_GROUP_AFFINITY = new CompoundName("dispatch.group.affinity"); - private final List<GroupSchedule> scoreboard; private int needle = 0; @@ -54,16 +51,7 @@ public class LoadBalancer { return Optional.empty(); } - Integer groupAffinity = query.properties().getInteger(QUERY_NODE_GROUP_AFFINITY); - if (groupAffinity != null) { - Optional<Group> previouslyChosen = allocateFromGroup(groupAffinity); - if (previouslyChosen.isPresent()) { - return previouslyChosen; - } - } - Optional<Group> allocatedGroup = allocateNextGroup(); - allocatedGroup.ifPresent(group -> query.properties().set(QUERY_NODE_GROUP_AFFINITY, group.id())); - return allocatedGroup; + return allocateNextGroup(); } /** @@ -83,18 +71,6 @@ public class LoadBalancer { } } - private Optional<Group> allocateFromGroup(int groupId) { - synchronized (this) { - for (GroupSchedule schedule : scoreboard) { - if (schedule.group.id() == groupId) { - schedule.adjustScore(1); - return Optional.of(schedule.group); - } - } - } - return Optional.empty(); - } - private Optional<Group> allocateNextGroup() { synchronized (this) { GroupSchedule bestSchedule = null; @@ -139,12 +115,12 @@ public class LoadBalancer { } public boolean isPreferredOver(GroupSchedule other) { - if (! group.hasSufficientCoverage()) { - return false; - } if (other == null) { return true; } + if (! group.hasSufficientCoverage()) { + return false; + } return this.score < other.score; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java index a0c7447fd3e..0d50702acfd 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java @@ -155,7 +155,7 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { /** Returns the n'th (zero-indexed) group in the cluster if possible */ public Optional<Group> group(int n) { - if (orderedGroups.size() < n) { + if (orderedGroups.size() > n) { return Optional.of(orderedGroups.get(n)); } else { return Optional.empty(); diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 81aa4a08f18..e340132a507 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -46,4 +46,16 @@ public class Coverage extends com.yahoo.container.handler.Coverage { public Coverage setNodesTried(int nodesTried) { super.setNodesTried(nodesTried); return this; } + public void mergeWithPartition(Coverage other) { + if (other == null) { + return; + } + int newResultSets = Integer.max(this.resultSets, other.resultSets); + int newFullResultSets = Integer.min(this.fullResultSets, other.fullResultSets); + + merge(other); + + this.resultSets = newResultSets; + this.fullResultSets = newFullResultSets; + } } diff --git a/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java b/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java index e64e0bc8f8d..21650d531be 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java +++ b/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java @@ -57,6 +57,11 @@ public class FieldComparator extends ChainableComparator { Object a = getField(first,fieldName); Object b = getField(second,fieldName); + // If either of the values are null, don't touch the ordering + // This is to avoid problems if the sorting is called before the + // result is filled. + if ((a == null) || (b == null)) return 0; + int x = compareValues(a, b, fieldOrder.getSorter()); if (x != 0) { if (fieldOrder.getSortOrder() == Sorting.Order.DESCENDING) @@ -76,12 +81,6 @@ public class FieldComparator extends ChainableComparator { @SuppressWarnings("rawtypes") private int compareValues(Object first, Object second, Sorting.AttributeSorter s) { - if (first == null) { - if (second == null) return 0; - return -1; - } else if (second == null) { - return 1; - } if (first.getClass().isInstance(second) && first instanceof Comparable) { // We now know: diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 483f55f9c92..c008b133595 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -16,7 +16,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Set; import java.util.stream.Collectors; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 5fa9dee8370..b08a3a73a01 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -80,27 +80,6 @@ public class LoadBalancerTest { } @Test - public void requreThatLoadBalancerReturnsSameGroupForSameQuery() { - Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); - Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); - - Query q = new Query(); - // get first group - Optional<Group> grp = lb.takeGroupForQuery(q); - Group group = grp.get(); - int id1 = group.id(); - // release allocation - lb.releaseGroup(group); - - // continue with same query - grp = lb.takeGroupForQuery(q); - group = grp.get(); - assertThat(group.id(), equalTo(id1)); - } - - @Test public void requreThatLoadBalancerReturnsGroupWithShortestQueue() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index bf5743e2d3c..940ae654b1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -1,17 +1,31 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.vespa.hosted.controller.api.authority.config.ApiAuthorityConfig; import com.yahoo.config.provision.SystemName; import com.yahoo.log.LogLevel; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Organization; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Contact; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Objects; @@ -29,44 +43,107 @@ public class ContactInformationMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(ContactInformationMaintainer.class.getName()); private final Organization organization; + private final String[] baseUris; + private CloseableHttpClient httpClient = HttpClientBuilder.create().build(); - public ContactInformationMaintainer(Controller controller, Duration interval, JobControl jobControl, Organization organization) { + public ContactInformationMaintainer(Controller controller, Duration interval, JobControl jobControl, Organization organization, ApiAuthorityConfig apiAuthorityConfig) { super(controller, interval, jobControl, null, EnumSet.of(SystemName.cd, SystemName.main)); this.organization = Objects.requireNonNull(organization, "organization must be non-null"); + this.baseUris = apiAuthorityConfig.authorities().split(","); } + // The maintainer will eventually feed contact info to systems other than its own, determined by the baseUris list. @Override protected void maintain() { - for (Tenant t : controller().tenants().asList()) { - if (!(t instanceof AthenzTenant)) continue; // No contact information for non-Athenz tenants - AthenzTenant tenant = (AthenzTenant) t; - if (!tenant.propertyId().isPresent()) continue; // Can only update contact information if property ID is known - try { - findContact(tenant).ifPresent(contact -> { - controller().tenants().lockIfPresent(t.name(), lockedTenant -> controller().tenants().store(lockedTenant.with(contact))); + for (String baseUri : baseUris) { + for (String tenantName : getTenantList(baseUri)) { + Optional<PropertyId> tenantPropertyId = getPropertyId(tenantName, baseUri); + if (!tenantPropertyId.isPresent()) + continue; + findContact(tenantPropertyId.get()).ifPresent(contact -> { + feedContact(tenantName, contact, baseUri); }); - } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to update contact information for " + tenant + ": " + - Exceptions.toMessageString(e) + ". Retrying in " + - maintenanceInterval()); } } } - /** Find contact information for given tenant */ - private Optional<Contact> findContact(AthenzTenant tenant) { - if (!tenant.propertyId().isPresent()) { - return Optional.empty(); + private void feedContact(String tenantName, Contact contact, String baseUri) { + try { + CloseableHttpClient httpClient = HttpClientBuilder.create().build(); + String uri = baseUri + "contactinfo/v1/tenant/" + tenantName; + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(contactToByteArrayEntity(contact)); + httpClient.execute(httpPost); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Failed to update contact information for " + tenantName + ": " + + Exceptions.toMessageString(e) + ". Retrying in " + + maintenanceInterval()); + } + } + + private ByteArrayEntity contactToByteArrayEntity(Contact contact) throws IOException { + Slime slime = new Slime(); + Cursor cursor = slime.setObject(); + cursor.setString("url", contact.url().toString()); + cursor.setString("issueTrackerUrl", contact.issueTrackerUrl().toString()); + cursor.setString("propertyUrl", contact.propertyUrl().toString()); + Cursor personsCursor = cursor.setArray("persons"); + for (List<String> personList : contact.persons()) { + Cursor sublist = personsCursor.addArray(); + for(String person : personList) { + sublist.addString(person); + } + } + return new ByteArrayEntity(SlimeUtils.toJsonBytes(slime)); + } + + private List<String> getTenantList(String baseUri) { + List<String> tenantList = new ArrayList<>(); + HttpGet getRequest = new HttpGet(baseUri + "application/v4/tenant/"); + try { + HttpResponse response = httpClient.execute(getRequest); + Slime slime = SlimeUtils.jsonToSlime(EntityUtils.toByteArray(response.getEntity())); + Inspector inspector = slime.get(); + inspector.traverse((ArrayTraverser) (index, tenant) -> { + tenantList.add(tenant.field("tenant").asString()); + }); + } catch (IOException e) { + log.log(LogLevel.WARNING, "Failed to get tenant list from base URI: " + baseUri.toString() + + Exceptions.toMessageString(e) + ". Retrying in " + + maintenanceInterval()); } - List<List<String>> persons = organization.contactsFor(tenant.propertyId().get()) + return tenantList; + } + + private Optional<PropertyId> getPropertyId(String tenantName, String baseUri) { + Optional<PropertyId> propertyId = Optional.empty(); + HttpGet getRequest = new HttpGet(baseUri + "application/v4/tenant/" + tenantName); + try { + HttpResponse response = httpClient.execute(getRequest); + Slime slime = SlimeUtils.jsonToSlime(EntityUtils.toByteArray(response.getEntity())); + Inspector inspector = slime.get(); + if (!inspector.field("propertyId").valid()) { + log.log(LogLevel.WARNING, "Unable to get property id for " + tenantName); + return Optional.empty(); + } + propertyId = Optional.of(new PropertyId(inspector.field("propertyId").asString())); + } catch (IOException e) { + log.log(LogLevel.WARNING, "Unable to get property idfor " + tenantName, e); + } + return propertyId; + } + + /** Find contact information for given tenant */ + private Optional<Contact> findContact(PropertyId propertyId) { + List<List<String>> persons = organization.contactsFor(propertyId) .stream() .map(personList -> personList.stream() .map(User::displayName) .collect(Collectors.toList())) .collect(Collectors.toList()); - return Optional.of(new Contact(organization.contactsUri(tenant.propertyId().get()), - organization.propertyUri(tenant.propertyId().get()), - organization.issueCreationUri(tenant.propertyId().get()), + return Optional.of(new Contact(organization.contactsUri(propertyId), + organization.propertyUri(propertyId), + organization.issueCreationUri(propertyId), persons)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index c67eab8826e..9fa300ba9b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; @@ -49,7 +50,7 @@ public class ControllerMaintenance extends AbstractComponent { private final ContactInformationMaintainer contactInformationMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, + public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, JobControl jobControl, Metric metric, Chef chefClient, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, @@ -72,7 +73,7 @@ public class ControllerMaintenance extends AbstractComponent { jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl); osUpgraders = osUpgraders(controller, jobControl); osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); - contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, organization); + contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, organization, apiAuthorityConfig); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/contactinfo/ContactInfoHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/contactinfo/ContactInfoHandler.java index 5c4573e1ce7..7651381b810 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/contactinfo/ContactInfoHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/contactinfo/ContactInfoHandler.java @@ -13,9 +13,6 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Organization; -import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.StringResponse; @@ -24,16 +21,11 @@ import com.yahoo.vespa.hosted.controller.tenant.Contact; import com.yahoo.yolean.Exceptions; import java.io.IOException; -import java.io.ObjectInputStream; import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; /** * This implements the contactinfo/v1 API which allows getting and feeding diff --git a/controller-server/src/main/resources/configdefinitions/api-authority.def b/controller-server/src/main/resources/configdefinitions/api-authority.def new file mode 100644 index 00000000000..e2feb221796 --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/api-authority.def @@ -0,0 +1,4 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.api.authority.config + +authorities string
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java index cbaa37b15e3..82fe061296e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java @@ -1,13 +1,17 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Contact; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import java.net.URI; @@ -18,9 +22,16 @@ import java.util.List; import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.findAll; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; /** * @author mpolden @@ -29,13 +40,27 @@ public class ContactInformationMaintainerTest { private ControllerTester tester; private ContactInformationMaintainer maintainer; + private String contactInfoPath = "/contactinfo/v1/tenant/tenant1"; + private String tenantPath = "/application/v4/tenant/"; + + @Rule + public WireMockRule wireMockRule = new WireMockRule(4443); + @Before public void before() { tester = new ControllerTester(); + ApiAuthorityConfig.Builder apiAuthorityConfigBuilder = new ApiAuthorityConfig.Builder().authorities("http://localhost:4443/"); + ApiAuthorityConfig apiAuthorityConfig = new ApiAuthorityConfig(apiAuthorityConfigBuilder); maintainer = new ContactInformationMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), - tester.organization()); + tester.organization(), apiAuthorityConfig); + wireMockRule.stubFor(post(urlEqualTo(contactInfoPath)) + .willReturn(aResponse().withStatus(200))); + wireMockRule.stubFor(get(urlEqualTo(tenantPath)) + .willReturn(okJson("[{\"tenant\":\"tenant1\"}]"))); + wireMockRule.stubFor(get(urlEqualTo(tenantPath + "tenant1")) + .willReturn(okJson("{\"tenant\":\"tenant1\", \"athensDomain\":\"domain\", \"property\":\"property\", \"propertyId\":\"1\"}"))); } @Test @@ -44,13 +69,13 @@ public class ContactInformationMaintainerTest { TenantName name = tester.createTenant("tenant1", "domain1", propertyId); Supplier<AthenzTenant> tenant = () -> tester.controller().tenants().requireAthenzTenant(name); assertFalse("No contact information initially", tenant.get().contact().isPresent()); - Contact contact = testContact(); registerContact(propertyId, contact); maintainer.run(); - - assertTrue("Contact information added", tenant.get().contact().isPresent()); - assertEquals(contact, tenant.get().contact().get()); + verify(1, postRequestedFor(urlEqualTo(contactInfoPath))); + LoggedRequest request = findAll(postRequestedFor(urlEqualTo(contactInfoPath))).get(0); + String expectedBody = "{\"url\":\"http://contact1.test\",\"issueTrackerUrl\":\"http://issue-tracker1.test\",\"propertyUrl\":\"http://property1.test\",\"persons\":[[\"alice\"],[\"bob\"]]}"; + assertEquals(expectedBody, new String(request.getBody())); } private void registerContact(long propertyId, Contact contact) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 9574256c065..cfd384998d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -80,7 +80,11 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'>\n" + + " <config name=\"vespa.hosted.controller.api.authority.config.api-authority\">\n" + + " <authorities>https://localhost:4443/</authorities>\n" + + " </config>" + + " </component>" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index f2e105f2c31..d620d78d3f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -47,13 +47,13 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; -import com.yahoo.vespa.hosted.controller.maintenance.ContactInformationMaintainer; import com.yahoo.vespa.hosted.controller.maintenance.DeploymentMetricsMaintainer; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; +import com.yahoo.vespa.hosted.controller.tenant.Contact; import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MultipartEntityBuilder; @@ -1269,9 +1269,8 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void updateContactInformation() { - new ContactInformationMaintainer(tester.controller(), Duration.ofDays(1), - new JobControl(tester.controller().curator()), - organization()).run(); + Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), URI.create("www.issues.tld/1234"), Arrays.asList(Arrays.asList("alice"), Arrays.asList("bob"))); + tester.controller().tenants().lockIfPresent(TenantName.from("tenant2"), lockedTenant -> tester.controller().tenants().store(lockedTenant.with(contact))); } private void registerContact(long propertyId) { diff --git a/fbench/src/fbench/client.cpp b/fbench/src/fbench/client.cpp index 0200ea2d067..754fc809511 100644 --- a/fbench/src/fbench/client.cpp +++ b/fbench/src/fbench/client.cpp @@ -8,13 +8,13 @@ #include <cassert> #include <cstring> -Client::Client(ClientArguments *args) +Client::Client(vespalib::CryptoEngine::SP engine, ClientArguments *args) : _args(args), _status(new ClientStatus()), _reqTimer(new Timer()), _cycleTimer(new Timer()), _masterTimer(new Timer()), - _http(new HTTPClient(_args->_hostname, _args->_port, + _http(new HTTPClient(std::move(engine), _args->_hostname, _args->_port, _args->_keepAlive, _args->_headerBenchmarkdataCoverage, _args->_extraHeaders, _args->_authority)), _reader(new FileReader()), diff --git a/fbench/src/fbench/client.h b/fbench/src/fbench/client.h index 1e0b750dbb2..ce7c13fc982 100644 --- a/fbench/src/fbench/client.h +++ b/fbench/src/fbench/client.h @@ -4,6 +4,7 @@ #include <fstream> #include <atomic> #include <thread> +#include <vespa/vespalib/net/crypto_engine.h> #define FBENCH_DELIMITER "\n[--xxyyzz--FBENCH_MAGIC_DELIMITER--zzyyxx--]\n" @@ -188,7 +189,7 @@ public: * The client arguments given to this method becomes the * responsibility of the client. **/ - Client(ClientArguments *args); + Client(vespalib::CryptoEngine::SP engine, ClientArguments *args); /** * Delete objects owned by this client, including the client arguments. diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp index c98c3ead4fe..36c60e08ef7 100644 --- a/fbench/src/fbench/fbench.cpp +++ b/fbench/src/fbench/fbench.cpp @@ -3,16 +3,40 @@ #include <httpclient/httpclient.h> #include <util/filereader.h> #include <util/clientstatus.h> +#include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/vespalib/net/tls/transport_security_options.h> +#include <vespa/vespalib/net/tls/tls_crypto_engine.h> +#include <vespa/vespalib/net/tls/crypto_exception.h> +#include <vespa/vespalib/io/mapped_file_input.h> #include "client.h" #include "fbench.h" #include <cstring> #include <cmath> #include <csignal> +namespace { + +std::string maybe_load(const std::string &file_name, bool &failed) { + std::string content; + if (!file_name.empty()) { + vespalib::MappedFileInput file(file_name); + if (file.valid()) { + content = std::string(file.get().data, file.get().size); + } else { + fprintf(stderr, "could not load file: '%s'\n", file_name.c_str()); + failed = true; + } + } + return content; +} + +} + sig_atomic_t exitSignal = 0; FBench::FBench() - : _clients(), + : _crypto_engine(), + _clients(), _ignoreCount(0), _cycle(0), _filenamePattern(NULL), @@ -35,6 +59,44 @@ FBench::~FBench() free(_outputPattern); } +bool +FBench::init_crypto_engine(const std::string &ca_certs_file_name, + const std::string &cert_chain_file_name, + const std::string &private_key_file_name) +{ + if (ca_certs_file_name.empty() && + cert_chain_file_name.empty() && + private_key_file_name.empty()) + { + _crypto_engine = std::make_shared<vespalib::NullCryptoEngine>(); + return true; + } + if (ca_certs_file_name.empty()) { + fprintf(stderr, "CA certificate required; specify with -T\n"); + return false; + } + if (cert_chain_file_name.empty() != private_key_file_name.empty()) { + fprintf(stderr, "both client certificate AND client private key required; specify with -C and -K\n"); + return false; + } + bool load_failed = false; + vespalib::net::tls::TransportSecurityOptions + tls_opts(maybe_load(ca_certs_file_name, load_failed), + maybe_load(cert_chain_file_name, load_failed), + maybe_load(private_key_file_name, load_failed)); + if (load_failed) { + fprintf(stderr, "failed to load transport security options\n"); + return false; + } + try { + _crypto_engine = std::make_shared<vespalib::TlsCryptoEngine>(tls_opts); + } catch (vespalib::net::tls::CryptoException &e) { + fprintf(stderr, "%s\n", e.what()); + return false; + } + return true; +} + void FBench::InitBenchmark(int numClients, int ignoreCount, int cycle, const char *filenamePattern, const char *outputPattern, @@ -78,7 +140,7 @@ FBench::CreateClients() off_beg = _queryfileOffset[i]; off_end = _queryfileOffset[i+1]; } - client = std::make_unique<Client>( + client = std::make_unique<Client>(_crypto_engine, new ClientArguments(i, _clients.size(), _filenamePattern, _outputPattern, _hostnames[i % _hostnames.size()].c_str(), _ports[i % _ports.size()], _cycle, @@ -226,12 +288,15 @@ FBench::Usage() printf(" -o <str> : save query results to output files with the given pattern\n"); printf(" (default is not saving.)\n"); printf(" -r <num> : number of times to re-use each query file. -1 means no limit [-1]\n"); - printf(" -m <num> : max line size in input query files [8192].\n"); + printf(" -m <num> : max line size in input query files [131072].\n"); printf(" Can not be less than the minimum [1024].\n"); printf(" -p <num> : print summary every <num> seconds.\n"); printf(" -k : disable HTTP keep-alive.\n"); - printf(" -y : write data on coverage to output file (must used with -x).\n"); - printf(" -z : use single query file to be distributed between clients.\n\n"); + printf(" -y : write data on coverage to output file.\n"); + printf(" -z : use single query file to be distributed between clients.\n"); + printf(" -T <str> : CA certificate file to verify peer against.\n"); + printf(" -C <str> : client certificate file name.\n"); + printf(" -K <str> : client private key file name.\n\n"); printf(" <hostname> : the host you want to benchmark.\n"); printf(" <port> : the port to use when contacting the host.\n\n"); printf("Several hostnames and ports can be listed\n"); @@ -263,6 +328,9 @@ FBench::Main(int argc, char *argv[]) const char *outputFilePattern = NULL; std::string queryStringToAppend; std::string extraHeaders; + std::string ca_certs_file_name; // -T + std::string cert_chain_file_name; // -C + std::string private_key_file_name; // -K int restartLimit = -1; bool keepAlive = true; @@ -282,7 +350,7 @@ FBench::Main(int argc, char *argv[]) idx = 1; optError = false; - while((opt = GetOpt(argc, argv, "H:A:a:n:c:l:i:s:q:o:r:m:p:kxyzP", arg, idx)) != -1) { + while((opt = GetOpt(argc, argv, "H:A:T:C:K:a:n:c:l:i:s:q:o:r:m:p:kxyzP", arg, idx)) != -1) { switch(opt) { case 'A': authority = arg; @@ -294,6 +362,15 @@ FBench::Main(int argc, char *argv[]) return -1; } break; + case 'T': + ca_certs_file_name = std::string(arg); + break; + case 'C': + cert_chain_file_name = std::string(arg); + break; + case 'K': + private_key_file_name = std::string(arg); + break; case 'a': queryStringToAppend = std::string(arg); break; @@ -365,6 +442,11 @@ FBench::Main(int argc, char *argv[]) return -1; } + if (!init_crypto_engine(ca_certs_file_name, cert_chain_file_name, private_key_file_name)) { + fprintf(stderr, "failed to initialize crypto engine\n"); + return -1; + } + short hosts = args / 2; for (int i=0; i<hosts; ++i) diff --git a/fbench/src/fbench/fbench.h b/fbench/src/fbench/fbench.h index 4c5885d8988..8cbab2e6d6c 100644 --- a/fbench/src/fbench/fbench.h +++ b/fbench/src/fbench/fbench.h @@ -10,6 +10,7 @@ class FBench { private: + vespalib::CryptoEngine::SP _crypto_engine; std::vector<Client::UP> _clients; int _numClients; int _ignoreCount; @@ -32,6 +33,10 @@ private: std::string _extraHeaders; std::string _authority; + bool init_crypto_engine(const std::string &ca_certs_file_name, + const std::string &cert_chain_file_name, + const std::string &private_key_file_name); + void InitBenchmark(int numClients, int ignoreCount, int cycle, const char *filenamePattern, const char *outputPattern, int byteLimit, int restartLimit, int maxLineSize, diff --git a/fbench/src/geturl/geturl.cpp b/fbench/src/geturl/geturl.cpp index f279b0b55c8..868d33f30a9 100644 --- a/fbench/src/geturl/geturl.cpp +++ b/fbench/src/geturl/geturl.cpp @@ -1,4 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/net/crypto_engine.h> #include <httpclient/httpclient.h> #include <iostream> @@ -10,7 +12,8 @@ main(int argc, char** argv) return -1; } - HTTPClient client(argv[1], atoi(argv[2]), false, false); + auto engine = std::make_shared<vespalib::NullCryptoEngine>(); + HTTPClient client(engine, argv[1], atoi(argv[2]), false, false); if (!client.Fetch(argv[3], &std::cout).Ok()) { fprintf(stderr, "geturl: could not fetch 'http://%s:%d%s'\n", argv[1], atoi(argv[2]), argv[3]); diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp index c49ef5da12c..002d2770dcd 100644 --- a/fbench/src/httpclient/httpclient.cpp +++ b/fbench/src/httpclient/httpclient.cpp @@ -17,10 +17,12 @@ HTTPClient::ChunkedReader HTTPClient::ChunkedReader::_instance; -HTTPClient::HTTPClient(const char *hostname, int port, +HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, int port, bool keepAlive, bool headerBenchmarkdataCoverage, const std::string & extraHeaders, const std::string &authority) - : _socket(new FastOS_Socket()), + : _engine(std::move(engine)), + _address(vespalib::SocketAddress::select_remote(port, hostname)), + _socket(), _hostname(hostname), _port(port), _keepAlive(keepAlive), @@ -48,7 +50,6 @@ HTTPClient::HTTPClient(const char *hostname, int port, _dataDone(false), _reader(NULL) { - _socket->SetAddressByHostName(port, hostname); if (_authority == "") { char tmp[1024]; snprintf(tmp, 1024, "%s:%d", hostname, port); @@ -56,17 +57,31 @@ HTTPClient::HTTPClient(const char *hostname, int port, } } +bool +HTTPClient::connect_socket() +{ + _socket.reset(); + auto handle = _address.connect([](auto &h) + { + return (h.set_nodelay(true) && + h.set_linger(false, 0)); + }); + if (!handle.valid()) { + return false; + } + _socket = vespalib::SyncCryptoSocket::create(*_engine, std::move(handle), false); + return bool(_socket); +} + ssize_t HTTPClient::FillBuffer() { - _bufused = _socket->Read(_buf, _bufsize); // may be -1 + _bufused = _socket->read(_buf, _bufsize); // may be -1 _bufpos = 0; return _bufused; } HTTPClient::~HTTPClient() { - if (_socket) - _socket->Close(); delete [] _buf; } @@ -148,9 +163,9 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen // try to reuse connection if keep-alive is enabled if (_keepAlive - && _socket->IsOpened() - && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req) - && (!usePost || _socket->Write(content, cLen) == (ssize_t)cLen) + && _socket + && _socket->write(req, strlen(req)) == (ssize_t)strlen(req) + && (!usePost || _socket->write(content, cLen) == (ssize_t)cLen) && FillBuffer() > 0) { // DEBUG @@ -161,17 +176,14 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen } return true; } else { - _socket->Close(); + _socket.reset(); ResetBuffer(); } // try to open new connection to server - if (_socket->SetSoBlocking(true) - && _socket->Connect() - && _socket->SetNoDelay(true) - && _socket->SetSoLinger(false, 0) - && _socket->Write(req, strlen(req)) == (ssize_t)strlen(req) - && (!usePost || _socket->Write(content, cLen) == (ssize_t)cLen)) + if (connect_socket() + && _socket->write(req, strlen(req)) == (ssize_t)strlen(req) + && (!usePost || _socket->write(content, cLen) == (ssize_t)cLen)) { // DEBUG @@ -181,7 +193,7 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen } return true; } else { - _socket->Close(); + _socket.reset(); } // DEBUG @@ -395,7 +407,7 @@ HTTPClient::ConnCloseReader::Read(HTTPClient &client, res = fromBuffer; } if ((len - fromBuffer) > (len >> 1)) { - readRes = client._socket->Read(static_cast<char *>(buf) + readRes = client._socket->read(static_cast<char *>(buf) + fromBuffer, len - fromBuffer); if (readRes < 0) { client.Close(); @@ -434,7 +446,7 @@ HTTPClient::ContentLengthReader::Read(HTTPClient &client, readLen = (len - fromBuffer < client._contentLength - client._dataRead) ? len - fromBuffer : client._contentLength - client._dataRead; - readRes = client._socket->Read(static_cast<char *>(buf) + readRes = client._socket->read(static_cast<char *>(buf) + fromBuffer, readLen); if (readRes < 0) { client.Close(); @@ -510,7 +522,7 @@ HTTPClient::Close() || _connectionCloseGiven || !_dataDone || (_httpVersion == 0 && !_keepAliveGiven)) ? - _socket->Close() : true; + (_socket.reset(), true) : true; } HTTPClient::FetchStatus diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h index 783545744d5..9c3ccd437d1 100644 --- a/fbench/src/httpclient/httpclient.h +++ b/fbench/src/httpclient/httpclient.h @@ -3,7 +3,9 @@ #include <ostream> #include <memory> -#include <vespa/fastos/socket.h> +#include <vespa/vespalib/net/sync_crypto_socket.h> +#include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/vespalib/net/socket_address.h> /** * This class implements a HTTP client that may be used to fetch @@ -88,7 +90,10 @@ protected: }; friend class HTTPClient::ChunkedReader; - std::unique_ptr<FastOS_Socket> _socket; + vespalib::CryptoEngine::SP _engine; + vespalib::SocketAddress _address; + vespalib::SyncCryptoSocket::UP _socket; + std::string _hostname; int _port; bool _keepAlive; @@ -131,6 +136,17 @@ protected: _bufused = 0; } + /** + * (re)connects the socket to the host/port specified in the + * constructor. The hostname is not resolved again; the resolve + * result is cached by the constructor. Also sets tcp nodelay flag + * and disables lingering. Note to servers: This is a no-nonsense + * socket that will be closed in your face in very ungraceful + * ways. Do not expect half-close niceties or tls session + * termination packets. + **/ + bool connect_socket(); + /** * Fill the internal buffer with data from the url we are connected * to. @@ -215,8 +231,8 @@ public: * @param port the TCP port to use when contacting the host. * @param keepAlive flag indicating if keep-alive should be enabled. **/ - HTTPClient(const char *hostname, int port, bool keepAlive, - bool headerBenchmarkdataCoverage, const std::string & extraHeaders="", const std::string &authority = ""); + HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname, int port, bool keepAlive, + bool headerBenchmarkdataCoverage, const std::string & extraHeaders="", const std::string &authority = ""); /** * Disconnect from server and free memory. diff --git a/fbench/src/test/httpclient.cpp b/fbench/src/test/httpclient.cpp index 4201da68b97..0d350c393c1 100644 --- a/fbench/src/test/httpclient.cpp +++ b/fbench/src/test/httpclient.cpp @@ -1,4 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/net/crypto_engine.h> #include <httpclient/httpclient.h> #include <iostream> #include <thread> @@ -11,13 +13,14 @@ main(int argc, char **argv) return 1; } + auto engine = std::make_shared<vespalib::NullCryptoEngine>(); HTTPClient *client; ssize_t len; if(argc == 4) { - client = new HTTPClient(argv[1], atoi(argv[2]), false, true); + client = new HTTPClient(engine, argv[1], atoi(argv[2]), false, true); } else { - client = new HTTPClient(argv[1], atoi(argv[2]), true, true); + client = new HTTPClient(engine, argv[1], atoi(argv[2]), true, true); } std::ostream * output = & std::cout; diff --git a/fbench/src/test/httpclient_splitstring.cpp b/fbench/src/test/httpclient_splitstring.cpp index d766b0f8f4b..4f3e0027db0 100644 --- a/fbench/src/test/httpclient_splitstring.cpp +++ b/fbench/src/test/httpclient_splitstring.cpp @@ -1,13 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/net/crypto_engine.h> #include <httpclient/httpclient.h> #include <cstring> class DebugHTTPClient : public HTTPClient { public: - DebugHTTPClient(const char* server, int port, bool keepAlive) - : HTTPClient(server, port, keepAlive, true) {} + DebugHTTPClient() + : HTTPClient(std::make_shared<vespalib::NullCryptoEngine>(), + "localhost", 80, true, true) {} static void SplitLineTest(const char *input); static void DebugSplitLine(); diff --git a/fsa/src/main/java/com/yahoo/fsa/FSA.java b/fsa/src/main/java/com/yahoo/fsa/FSA.java index a4ec93286c4..a0ca7ebee36 100644 --- a/fsa/src/main/java/com/yahoo/fsa/FSA.java +++ b/fsa/src/main/java/com/yahoo/fsa/FSA.java @@ -17,7 +17,7 @@ import java.util.NoSuchElementException; /** * Finite-State Automaton. * - * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + * @author Peter Boros */ public class FSA { @@ -158,6 +158,7 @@ public class FSA { * Class used to iterate over all accepted strings in the fsa. */ public static class Iterator implements java.util.Iterator<Iterator.Item> { + /** * Internally, this class stores the state information for the iterator. * Externally, it is used for accessing the data associated with the iterator position. @@ -171,6 +172,7 @@ public class FSA { /** * Constructor + * * @param fsa the FSA object the iterator is associated with. * @param state the state used as start state. */ @@ -216,6 +218,7 @@ public class FSA { return fsa.dataString(state); } + @Override public String toString() { return "string: " + string + "(" + getString() + "), symbol: " + symbol + ", state: " + state; } @@ -382,25 +385,19 @@ public class FSA { private void init(FileInputStream file, String charsetname) { try { _charset = Charset.forName(charsetname); - _header = file.getChannel().map(MapMode.READ_ONLY,0,256); _header.order(ByteOrder.LITTLE_ENDIAN); if (h_magic()!=2038637673) { throw new IOException("Stream does not contain an FSA: Wrong file magic number " + h_magic()); } - _symbol_tab = file.getChannel().map(MapMode.READ_ONLY, - 256,h_size()); + _symbol_tab = file.getChannel().map(MapMode.READ_ONLY, 256, h_size()); _symbol_tab.order(ByteOrder.LITTLE_ENDIAN); - _state_tab = file.getChannel().map(MapMode.READ_ONLY, - 256+h_size(),4*h_size()); + _state_tab = file.getChannel().map(MapMode.READ_ONLY, 256+h_size(), 4*h_size()); _state_tab.order(ByteOrder.LITTLE_ENDIAN); - _data = file.getChannel().map(MapMode.READ_ONLY, - 256+5*h_size(),h_data_size()); + _data = file.getChannel().map(MapMode.READ_ONLY, 256+5*h_size(), h_data_size()); _data.order(ByteOrder.LITTLE_ENDIAN); if(h_has_phash()>0){ - _phash = file.getChannel().map(MapMode.READ_ONLY, - 256+5*h_size()+h_data_size(), - 4*h_size()); + _phash = file.getChannel().map(MapMode.READ_ONLY, 256+5*h_size()+h_data_size(), 4*h_size()); _phash.order(ByteOrder.LITTLE_ENDIAN); } _ok=true; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java index 68be5bb5d94..f28731a47c7 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ApplicationLoader.java @@ -101,7 +101,7 @@ public class ApplicationLoader implements BootstrapLoader, ContainerActivator, C return; // application class bound by another module } try { - final Class<Application> appClass = ContainerBuilder.safeClassCast(Application.class, Class.forName(appLocation)); + Class<Application> appClass = ContainerBuilder.safeClassCast(Application.class, Class.forName(appLocation)); guiceModules.install(new AbstractModule() { @Override public void configure() { diff --git a/jrt/src/com/yahoo/jrt/XorCryptoSocket.java b/jrt/src/com/yahoo/jrt/XorCryptoSocket.java index 90be58bb700..fa784f62b12 100644 --- a/jrt/src/com/yahoo/jrt/XorCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/XorCryptoSocket.java @@ -1,20 +1,21 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SocketChannel; +import java.security.SecureRandom; import java.util.ArrayDeque; import java.util.Queue; import java.util.Random; - /** * A very simple CryptoSocket that performs connection handshaking and * data transformation. Used to test encryption integration separate * from TLS. - **/ + * + * @author havardpe + */ public class XorCryptoSocket implements CryptoSocket { private static final int CHUNK_SIZE = 4096; @@ -28,7 +29,7 @@ public class XorCryptoSocket implements CryptoSocket { private SocketChannel channel; private static byte genKey() { - return (byte) new Random().nextInt(256); + return (byte) new SecureRandom().nextInt(256); } private HandshakeResult readKey() throws IOException { diff --git a/libmlr/OWNERS b/libmlr/OWNERS deleted file mode 100644 index 6b09ce48bd4..00000000000 --- a/libmlr/OWNERS +++ /dev/null @@ -1 +0,0 @@ -lesters diff --git a/libmlr/README b/libmlr/README deleted file mode 100644 index 22821d78762..00000000000 --- a/libmlr/README +++ /dev/null @@ -1,2 +0,0 @@ -Tool to convert from GBDT to Vespa ranking epxressions. - diff --git a/libmlr/bin/xml2cpp b/libmlr/bin/xml2cpp deleted file mode 100755 index b92bd8d4a65..00000000000 --- a/libmlr/bin/xml2cpp +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findhost () { - if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 - fi - validate="${VESPA_HOME}/bin/vespa-validate-hostname" - if [ -f "$validate" ]; then - "$validate" "${VESPA_HOSTNAME}" || exit 1 - fi - export VESPA_HOSTNAME -} - -findroot -findhost - -# END environment bootstrap section - -if [ $# -lt 2 ]; then - echo "USAGE $0 <template> <mlrxxx.xml>"; - exit; -fi - -JAVA=java -CLASSPATH=$VESPA_HOME/lib/jars/xml2cpp.jar - -echo ${JAVA} -Xms64m -Xmx256m -cp ${CLASSPATH} com.yahoo.yst.libmlr.converter.DecisionTreeXmlToCpp -h $1 -i $2 -${JAVA} -Xms64m -Xmx256m -cp ${CLASSPATH} com.yahoo.yst.libmlr.converter.DecisionTreeXmlToCpp -h $1 -i $2 diff --git a/libmlr/pom.xml b/libmlr/pom.xml deleted file mode 100644 index 05b17d7ba50..00000000000 --- a/libmlr/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>xml2cpp</artifactId> - <packaging>jar</packaging> - <version>1.0.0-SNAPSHOT</version> - <name>xml2cpp</name> - <description>Fork of xml2cppConverver with support for SS3 models.</description> - <dependencies> - </dependencies> - <build> - <finalName>${project.artifactId}</finalName> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <version>2.5</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.6.1</version> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Xlint:-serial</arg> - <arg>-Werror</arg> - </compilerArgs> - <showWarnings>true</showWarnings> - <source>1.8</source> - <target>1.8</target> - <showDeprecation>true</showDeprecation> - <showWarnings>true</showWarnings> - <optimize>true</optimize> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <version>2.5.2</version> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - </plugins> - </build> - <properties> - <test.hide>true</test.hide> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> -</project> diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/DecisionTreeXmlToCpp.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/DecisionTreeXmlToCpp.java deleted file mode 100644 index 98eef506194..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/DecisionTreeXmlToCpp.java +++ /dev/null @@ -1,539 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.text.MessageFormat; -import java.util.Date; -import java.util.HashSet; - -import com.yahoo.yst.libmlr.converter.entity.EarlyExit; -import com.yahoo.yst.libmlr.converter.entity.Epilog; -import com.yahoo.yst.libmlr.converter.entity.FuncNormalize; -import com.yahoo.yst.libmlr.converter.entity.FuncPolytransform; -import com.yahoo.yst.libmlr.converter.entity.Function; -import com.yahoo.yst.libmlr.converter.entity.InternalNode; -import com.yahoo.yst.libmlr.converter.entity.ResponseNode; -import com.yahoo.yst.libmlr.converter.entity.Tree; -import com.yahoo.yst.libmlr.converter.entity.TreeNode; -import com.yahoo.yst.libmlr.converter.entity.TreeNodeVisitor; -import com.yahoo.yst.libmlr.converter.entity.TreenetFunction; -import com.yahoo.yst.libmlr.converter.parser.DecisionTreeXmlException; -import com.yahoo.yst.libmlr.converter.parser.MlrXmlParser; - -/** - * This class generates C++ from an MLR Decision Tree File - * - * @author allenwei - * - */ -public class DecisionTreeXmlToCpp { - - private static final String INDENT_UNIT = " "; - - private TreenetFunction tnFunc; - private String strCppFile; - private PrintWriter fpCpp; - - private int nodeIdx; - private int gIndentLevel; // global indent level - - public void setTnFunc(TreenetFunction tnFunc) { - this.tnFunc = tnFunc; - } - - public DecisionTreeXmlToCpp(String file) { - strCppFile = file; - try { - fpCpp = new PrintWriter( - new BufferedWriter( - new FileWriter(strCppFile))); - } catch (IOException ioex) { - System.out.println("Cannot open " + strCppFile + " for write"); - } - } - - /** - * Generates C++ code. - */ - public void genCode(String strHeaderFile) { - genCodeHeader(strHeaderFile); - gIndentLevel = 0; - setNodeIndex(); - genCodeDefs(); - genCodeFunc(); - } - - private void genCodeHeader(String strHeaderFile) { - String fmt = getFormatString(strHeaderFile); - String fileName = "mlr" + tnFunc.getFunctionId() + ".c"; - int nTrees = tnFunc.getNumberOfTrees(); - int nLeaves = tnFunc.getTree(0).getNumInternalNodes() + 1; - String header = MessageFormat.format(fmt, fileName, new Date(), - Integer.toString(nTrees), - Integer.toString(nLeaves)); - - gIndentLevel = 0; - printLn(0, header); - } - - private void setNodeIndex() { - - // set node id for each tree - int n = tnFunc.getNumberOfTrees(); - - SetNodeIndexVisitor nodeVisitor = new SetNodeIndexVisitor(); - SetLeafIndexVisitor leafVisitor = new SetLeafIndexVisitor(); - for (int i = 0; i < n; i++) { - nodeIdx = 0; - Tree tree = tnFunc.getTree(i); - traverseTree(tree.getRoot(), nodeVisitor); - traverseTree(tree.getRoot(), leafVisitor); - } - } - - private void genCodeDefs() { - printLn(0, "#define TOTAL_TREES " + tnFunc.getNumberOfTrees()); - printLn(); - - // const def for internal node labels - //genCodeTraverseTrees(0, null, new PrintNodeLabelDefVisitor(), null); - - // const def for leaf node labels - //genCodeTraverseTrees(0, null, new PrintLeafLabelDefVisitor(), null); - - genCodeNamespaceDefs(); - - // array init for internal nodes - genCodeTraverseTrees(1, "static const TreeNode nodes[] = {", - new PrintNodeInitVisitor(), "};"); - - // array init for leaf nodes - genCodeTraverseTrees(1, "static const double leaves[] = {", - new PrintLeafInitVisitor(), "};"); - - // array of tree size (number of internal nodes) - genCodeTreeSizeArrayInit(1); - - genCodeEarlyExits(1); - - } - - private void genCodeNamespaceDefs() { - printLn(0, "namespace " + tnFunc.getNameSpace() + " {"); - printLn(); - printLn(0, "enum Feature {"); - for (String f : tnFunc.getFeatureSet()) { - printLn(1, f + ","); - } - printLn(1, "NUMBER_FEATURES"); - printLn(0, "}; /* enum */"); // end enum - printLn(); - printLn(0, "} /* namespace */"); // end namespace - printLn(); - } - - private void genCodeFunc() { - - // function definition - printLn(0, "double"); - printLn(0, tnFunc.getFunctionName() + "(MlrScoreReq& msr) {"); - printLn(); - - genFeatureArrayDecl(1); - - // call traverseAll() - printLn(1, "msr.traverseAll(nodes, leaves, fValue, TOTAL_TREES, numNodes, meExits);"); - printLn(); - - genCodeEpilog(1); - - printLn(1, "return msr.getScore();"); - printLn(0, "}"); // end function - - fpCpp.close(); - } - - private void genFeatureArrayDecl(int indentInc) { - String ns = tnFunc.getNameSpace(); - - printLn(indentInc, "double fValue[" + ns + "::NUMBER_FEATURES];"); - printLn(); - - // FNTM: Distinguished values - //printInd(1, "double FNTM = fValue[" + ns + "::FNTM] = msr.getFeature(rf::FNTM);"); - //printLn(); - - HashSet<String> fSet = tnFunc.getFeatureSet(); - //fSet.remove("FNTM"); - - // initialization of features - for (String f : fSet) { - printLn(indentInc, "fValue[" + ns + "::" + f + "] = msr.getFeature(rf::" + f + ");"); - } - - printLn(); - } - - /** - * Prints code by iterating over all trees and visiting each tree node with - * the TreeNodeVisitor. - * - * @param indentInc - - * indentation level of the first line - * @param start - - * code printed before iterations - * @param end - - * code printed after iterations - */ - private void genCodeTraverseTrees(int indentInc, String start, - TreeNodeVisitor visitor, String end) { - - if (start != null) - printLn(indentInc, start); - - gIndentLevel += (indentInc + 1); - int n = tnFunc.getNumberOfTrees(); - for (int i = 0; i < n; i++) { - Tree t = tnFunc.getTree(i); - printLn("// " + t.getId() + " " + t.getComment()); - - traverseTree(t.getRoot(), visitor); - printLn(); - } - gIndentLevel -= (indentInc + 1); - - if (end != null) - printLn(indentInc, end); - - printLn(); - } - - private void genCodeTreeSizeArrayInit(int indentInc) { - String strDef = "static const int numNodes[" - + tnFunc.getNumberOfTrees() + "] = {"; - printLn(indentInc, strDef); - - int n = tnFunc.getNumberOfTrees(); - for (int i = 0; i < n; i++) { - String msg = tnFunc.getTree(i).getNumInternalNodes() + ", // " + i; - printLn(indentInc + 1, msg); - } - - printLn(indentInc, "};"); - printLn(); - } - - private void genCodeEarlyExits(int indentInc) { - printLn(indentInc, "static const MlrEarlyExit meExits[] = {"); - - int n = tnFunc.getNumEarlyExits(); - for (int i = 0; i < n; i++) { - EarlyExit eex = tnFunc.getEarlyExit(i); - String strEarlyExit = "{" + eex.getTreeId() + ", " - + "decisiontree::OP_" + eex.getOperator().getId().toUpperCase() + ", " - + eex.getValue() - + "},"; - printLn(indentInc + 1, strEarlyExit); - } - - // always generate a sentinel element for terminal condition - String sentinel = - "{" + tnFunc.getNumberOfTrees() - + ", decisiontree::OP_NONE, 0.0}"; - printLn(indentInc + 1, sentinel); - printLn(indentInc, "};"); - printLn(); - } - - /** - * Currently only generate code for normalize() - */ - private void genCodeEpilog(int indentInc) { - Epilog epilog = tnFunc.getEpilog(); - if (epilog == null) - return; - - Function func = epilog.getFunction(); - - if (func instanceof FuncNormalize) { - FuncNormalize funcNorm = (FuncNormalize) func; - - if (funcNorm.getInvertMethod() != FuncNormalize.INV_NONE) { - genCodeInversion(indentInc, funcNorm); - } - - if (funcNorm.isGenNormalize()) { - genCodeNormalize(indentInc, funcNorm); - } - } else if (func instanceof FuncPolytransform) { - FuncPolytransform funcPolytransform = (FuncPolytransform) func; - genCodePolytransform(indentInc, funcPolytransform); - } else { - throw new MlrCodeGenException("Unknown <epilogue> function: " + func.getClass().getName()); - } - } - - private void genCodeInversion(int indentInc, FuncNormalize funcNorm) { - if (funcNorm.getInvertMethod() == FuncNormalize.INV_INVERSION) { - printLn(indentInc, "msr.invert(" + funcNorm.getInvertedFrom() + ");"); - printLn(); - } else if (funcNorm.getInvertMethod() == FuncNormalize.INV_NEGATION) { - printLn(indentInc, "msr.negate();"); - printLn(); - } - } - - private void genCodeNormalize(int indentInc, FuncNormalize funcNorm) { - StringBuilder sb = new StringBuilder(); - - printLn(indentInc, "msr.normalize("); - - sb.append(funcNorm.getMean0()).append(", ") - .append(funcNorm.getSd0()).append(", ") - .append(funcNorm.getA0()).append(", ") - .append(funcNorm.getB0()).append(", "); - printLn(indentInc + 1, sb.toString()); - - sb.setLength(0); - sb.append(funcNorm.getMean1()).append(", ") - .append(funcNorm.getSd1()).append(", ") - .append(funcNorm.getA1()).append(", ") - .append(funcNorm.getB1()).append(", "); - printLn(indentInc + 1, sb.toString()); - - sb.setLength(0); - sb.append(funcNorm.getMean2()).append(", ") - .append(funcNorm.getSd2()).append(", ") - .append(funcNorm.getA2()).append(", ") - .append(funcNorm.getB2()).append(", "); - printLn(indentInc + 1, sb.toString()); - - sb.setLength(0); - sb.append(funcNorm.getMean3()).append(", ") - .append(funcNorm.getSd3()).append(", ") - .append(funcNorm.getA3()).append(", ") - .append(funcNorm.getB3()); - printLn(indentInc + 1, sb.toString()); - - printLn(indentInc, ");"); - printLn(); - - } - - private void genCodePolytransform(int indentInc, FuncPolytransform funcPoly) { - StringBuilder sb = new StringBuilder(); - sb.append("msr.polytransform("); - sb.append(funcPoly.getA0()).append(", ") - .append(funcPoly.getA1()).append(", ") - .append(funcPoly.getA2()).append(", ") - .append(funcPoly.getA3()).append(");"); - printLn(indentInc, sb.toString()); - printLn(); - - } - - // Utilities - - private String getFormatString(String strFmtFile) { - try { - BufferedReader fp = new BufferedReader( - new FileReader(strFmtFile)); - StringBuilder sb = new StringBuilder(); - String line = null; - while ((line = fp.readLine()) != null) { - sb.append(line).append("\n"); - } - - String fmt = sb.toString(); - fp.close(); - - return fmt; - - } catch (FileNotFoundException e) { - throw new MlrCodeGenException(strFmtFile, e); - } catch (IOException ioe) { - throw new MlrCodeGenException("reading file " + strFmtFile, - ioe); - } - } - - private void traverseTree(TreeNode node, TreeNodeVisitor v) { - v.visit(node); - if (node instanceof InternalNode) { - InternalNode dcNode = (InternalNode) node; - traverseTree(dcNode.getLeftNode(), v); - traverseTree(dcNode.getRightNode(), v); - } - } - - private void printAppend(String str) { - fpCpp.print(str); - } - - private void printLn(int inc, String str) { - int indent = gIndentLevel + inc; - for (int i = 0; i < indent; i++) { - fpCpp.print(INDENT_UNIT); - } - fpCpp.println(str); - } - - private void printLn(String str) { - printLn(0, str); - } - - private void printLn() { - fpCpp.println(); - } - - /** - * subclasses of TreeNodeVisitor - */ - - private class SetNodeIndexVisitor implements TreeNodeVisitor { - - public void visit(TreeNode node) { - if (node instanceof InternalNode) { - node.setIndex(nodeIdx++); - } - } - } - - private class SetLeafIndexVisitor implements TreeNodeVisitor { - - public void visit(TreeNode node) { - if (node instanceof ResponseNode) { - node.setIndex(nodeIdx++); - } - } - } - - /* - private class PrintNodeLabelDefVisitor implements TreeNodeVisitor { - - public void visit(TreeNode node) { - if (node instanceof InternalNode) { - printInd(0, "#define " + node.label + " " + node.nodeId); - } - } - } - - private class PrintLeafLabelDefVisitor implements TreeNodeVisitor { - - public void visit(TreeNode node) { - if (node instanceof LeafNode) { - printInd(0, "#define " + node.label + " " + node.nodeId); - } - } - } - */ - - private class PrintNodeInitVisitor implements TreeNodeVisitor { - - public void visit(TreeNode treeNode) { - if (treeNode instanceof InternalNode) { - InternalNode node = (InternalNode) treeNode; - int leftNodeIndex = node.getLeftNode().getIndex(); - int rightNodeIndex = node.getRightNode().getIndex(); - - StringBuilder sb = new StringBuilder(); - sb.append(node.getIndex() + " " + node.getId()); - sb.append(" ").append(node.getComment()); - - String str = "{ " + tnFunc.getNameSpace() + "::" + node.getFeature() + ", " - + node.getValue() + ", " + leftNodeIndex + ", " - + rightNodeIndex + " }, // " + sb.toString(); - - printLn(str); - } - } - } - - private class PrintLeafInitVisitor implements TreeNodeVisitor { - - public void visit(TreeNode treeNode) { - if (treeNode instanceof ResponseNode) { - ResponseNode node = (ResponseNode) treeNode; - StringBuilder sb = new StringBuilder(); - sb.append(node.getIndex() + " " + node.getId()); - sb.append(" ").append(node.getComment()); - - String str = node.getResponse() + ", // " + sb.toString(); - printLn(str); - } - } - } - - public static void main(String[] args) { - String xmlFile = null; - String headerFile = null; - String cppFile = null; - - int i = 0; - boolean hasErrors = false; - while (i < args.length && !hasErrors) { - String arg = args[i++]; - if (arg.equals("-i")) { - if (i < args.length) - xmlFile = args[i++]; - else - hasErrors = true; - - } else if (arg.equals("-h")) { - if (i < args.length) - headerFile = args[i++]; - else - hasErrors = true; - - } else if (arg.equals("-o")) { - if (i < args.length) - cppFile = args[i++]; - else - hasErrors = true; - } - - } - - if (xmlFile == null || headerFile == null) - hasErrors = true; - - if (hasErrors) { - System.out.println("USAGE: java DecisionTreeXmlToCpp -i XML_file -h header_file [-o Cpp_file]"); - return; - } - - if (cppFile == null) { - if (xmlFile.endsWith(".xml")) { - int idx = xmlFile.lastIndexOf('.'); - cppFile = xmlFile.substring(0, idx+1) + "c"; - } else { - cppFile = xmlFile + ".c"; - } - } - - File fpCpp = new File(cppFile); - if (fpCpp.exists()) { - System.out.println(cppFile + " exits. Please rename and run again."); - return; - } - - try { - MlrXmlParser parser = new MlrXmlParser(); - DecisionTreeXmlToCpp toCpp = new DecisionTreeXmlToCpp(cppFile); - - toCpp.setTnFunc((TreenetFunction) parser.parseXmlFile(xmlFile)); - toCpp.genCode(headerFile); - - } catch (DecisionTreeXmlException tnex) { - tnex.printStackTrace(); - } - - } -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/MlrCodeGenException.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/MlrCodeGenException.java deleted file mode 100644 index ca28e0bde8c..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/MlrCodeGenException.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter; - -public class MlrCodeGenException extends RuntimeException { - - public MlrCodeGenException() { - super(); - } - - public MlrCodeGenException(String msg) { - super(msg); - } - - public MlrCodeGenException(String msg, Throwable cause) { - super(msg, cause); - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/XmlUtils.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/XmlUtils.java deleted file mode 100644 index f0d7edff529..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/XmlUtils.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter; - -import java.util.ArrayList; - -import org.w3c.dom.Element; -import org.w3c.dom.Node; - -public class XmlUtils { - - public static Element getFirstChildElement(Node parent) { - if (parent == null) - return null; - - Node nd = parent.getFirstChild(); - while (nd != null) { - //System.out.println("type: " + nd.getNodeType() + " name: " + nd.getNodeName()); - if (nd.getNodeType() == Node.ELEMENT_NODE) { - return (Element)nd; - } - nd = nd.getNextSibling(); - } - - return null; - } - - public static Element getFirstChildElementByName(Node parent, String childName) { - if (parent == null) - return null; - - Node nd = parent.getFirstChild(); - while (nd != null) { - //System.out.println("type: " + nd.getNodeType() + " name: " + nd.getNodeName()); - if (nd.getNodeType() == Node.ELEMENT_NODE - && nd.getNodeName().equals(childName)) { - return (Element)nd; - } - nd = nd.getNextSibling(); - } - - return null; - } - - public static ArrayList<Element> getChildrenByName(Node parent, String childName) { - if (parent == null) - return null; - - ArrayList<Element> list = new ArrayList<Element>(); - Node nd = parent.getFirstChild(); - while (nd != null) { - //System.out.println("type: " + nd.getNodeType() + " name: " + nd.getNodeName()); - if (nd.getNodeType() == Node.ELEMENT_NODE - && nd.getNodeName().equals(childName)) { - list.add((Element)nd); - } - nd = nd.getNextSibling(); - } - - if (list.size() == 0) - return null; - else - return list; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/EarlyExit.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/EarlyExit.java deleted file mode 100644 index 010680c2e95..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/EarlyExit.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - - -public class EarlyExit { - protected int treeId; - protected Operator operator; - protected String value; - - public EarlyExit(int tid, Operator op, String val) { - treeId = tid; - operator = op; - value = val; - } - - public int getTreeId() { - return treeId; - } - - public String getValue() { - return value; - } - - public Operator getOperator() { - return operator; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Epilog.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Epilog.java deleted file mode 100644 index 45f1ddf20a4..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Epilog.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public class Epilog { - - protected Function function; - - public Function getFunction() { - return function; - } - - public void setFunction(Function f) { - this.function = f; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/FuncNormalize.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/FuncNormalize.java deleted file mode 100644 index a783dc42d09..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/FuncNormalize.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public class FuncNormalize implements Function { - - public static int INV_NONE = 0; - public static int INV_INVERSION = 1; - public static int INV_NEGATION = 2; - - private int invertMethod = INV_NONE; - private String invertedFrom; - - /* - * The following parameters are type String to preserve precision. - */ - private boolean pGenNormalize; - protected String mean0; - protected String mean1; - protected String mean2; - protected String mean3; - protected String sd0; - protected String sd1; - protected String sd2; - protected String sd3; - protected String a0; - protected String a1; - protected String a2; - protected String a3; - protected String b0; - protected String b1; - protected String b2; - protected String b3; - - public FuncNormalize() {} - - public int getInvertMethod() { - return invertMethod; - } - - public void setInvertMethod(int inv) { - this.invertMethod = inv; - } - - public String getInvertedFrom() { - return invertedFrom; - } - - public void setInvertedFrom(String invertedFrom) { - this.invertedFrom = invertedFrom; - } - - public boolean isGenNormalize() { - return pGenNormalize; - } - - public void setpDoNormalize(boolean doNormalize) { - this.pGenNormalize = doNormalize; - } - - public String getMean0() { - return mean0; - } - - public void setMean0(String mean0) { - this.mean0 = mean0; - } - - public String getMean1() { - return mean1; - } - - public void setMean1(String mean1) { - this.mean1 = mean1; - } - - public String getMean2() { - return mean2; - } - - public void setMean2(String mean2) { - this.mean2 = mean2; - } - - public String getMean3() { - return mean3; - } - - public void setMean3(String mean3) { - this.mean3 = mean3; - } - - public String getSd0() { - return sd0; - } - - public void setSd0(String sd0) { - this.sd0 = sd0; - } - - public String getSd1() { - return sd1; - } - - public void setSd1(String sd1) { - this.sd1 = sd1; - } - - public String getSd2() { - return sd2; - } - - public void setSd2(String sd2) { - this.sd2 = sd2; - } - - public String getSd3() { - return sd3; - } - - public void setSd3(String sd3) { - this.sd3 = sd3; - } - - public String getA0() { - return a0; - } - - public void setA0(String a0) { - this.a0 = a0; - } - - public String getA1() { - return a1; - } - - public void setA1(String a1) { - this.a1 = a1; - } - - public String getA2() { - return a2; - } - - public void setA2(String a2) { - this.a2 = a2; - } - - public String getA3() { - return a3; - } - - public void setA3(String a3) { - this.a3 = a3; - } - - public String getB0() { - return b0; - } - - public void setB0(String b0) { - this.b0 = b0; - } - - public String getB1() { - return b1; - } - - public void setB1(String b1) { - this.b1 = b1; - } - - public String getB2() { - return b2; - } - - public void setB2(String b2) { - this.b2 = b2; - } - - public String getB3() { - return b3; - } - - public void setB3(String b3) { - this.b3 = b3; - } - - public boolean validateParams() { - if (mean0 != null) { - if (mean1 == null || mean2 == null || mean3 == null || - sd0 == null || sd1 == null || sd2 == null || sd3 == null || - a0 == null || a1 == null || a2 == null || a3 == null || - b0 == null || b1 == null || b2 == null || b3 == null) { - return false; - } else { - pGenNormalize = true; - } - } else { // mean0 == null - if (mean1 != null || mean2 != null || mean3 != null || - sd0 != null || sd1 != null || sd2 != null || sd3 != null || - a0 != null || a1 != null || a2 != null || a3 != null || - b0 != null || b1 != null || b2 != null || b3 != null) { - return false; - } else { - pGenNormalize = false; - } - } - - return true; - } -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/FuncPolytransform.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/FuncPolytransform.java deleted file mode 100644 index 2934502b46e..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/FuncPolytransform.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public class FuncPolytransform implements Function { - /* - * The following parameters are type String to preserve precision. - */ - protected String a0; - protected String a1; - protected String a2; - protected String a3; - - public FuncPolytransform() {} - - public String getA0() { - return a0; - } - - public void setA0(String a0) { - this.a0 = a0; - } - - public String getA1() { - return a1; - } - - public void setA1(String a1) { - this.a1 = a1; - } - - public String getA2() { - return a2; - } - - public void setA2(String a2) { - this.a2 = a2; - } - - public String getA3() { - return a3; - } - - public void setA3(String a3) { - this.a3 = a3; - } - - public boolean validateParams() { - return (a0 != null && a1 != null && a2 != null && a3 != null); - } -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Function.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Function.java deleted file mode 100644 index 95bfe7cfc7d..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Function.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public interface Function { - - public boolean validateParams(); - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/InternalNode.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/InternalNode.java deleted file mode 100644 index 058235ec3b6..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/InternalNode.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - - -public class InternalNode extends TreeNode { - - private String feature; - private String op; - private String value; - private TreeNode left; // true - private TreeNode right; // false - - public InternalNode(String i, String c, String f, String v, TreeNode lf, TreeNode rt) { - super(i, c); - feature = f; - value = v; - left = lf; - right = rt; - } - - public String getFeature() { - return feature; - } - - public void setFeature(String feature) { - this.feature = feature; - } - - public String getOp() { - return op; - } - - public void setOp(String op) { - this.op = op; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public TreeNode getLeftNode() { - return left; - } - - public void setLeftNode(TreeNode left) { - this.left = left; - } - - public TreeNode getRightNode() { - return right; - } - - public void setRightNode(TreeNode right) { - this.right = right; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/MlrFunction.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/MlrFunction.java deleted file mode 100644 index f3a7c1f8677..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/MlrFunction.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - - -public abstract class MlrFunction { - protected String functionName; - protected String funcId; // numeric function id - protected String featureDefFile; - protected Epilog epilog; - - public String getFunctionName() { - return functionName; - } - - public String getFunctionId() { - return funcId; - } - - public String getFeatureDefFile() { - return featureDefFile; - } - - public Epilog getEpilog() { - return epilog; - } - - public void setEpilog(Epilog epilog) { - this.epilog = epilog; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Operator.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Operator.java deleted file mode 100644 index b28d5d1f0b6..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Operator.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public enum Operator { - EQ("eq"), - NEQ("neq"), - GT("gt"), - GEQ("geq"), - LT("lt"), - LEQ("leq"); - - private final String id; - - Operator(String id) { - this.id = id; - } - - public static Operator parse(String str) { - for (Operator op : Operator.values()) { - if (op.id.equals(str)) - return op; - } - throw new IllegalArgumentException(); - } - - public String getId() { - return id; - } - - public static void main(String[] args) { - Operator op = Operator.parse("gt"); - System.out.println("operator.toString = " + op.toString()); - System.out.println("operator = " + op.getId()); - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/ResponseNode.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/ResponseNode.java deleted file mode 100644 index fa282958cac..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/ResponseNode.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public class ResponseNode extends TreeNode { - - private double response; - - public ResponseNode(String i, String c, double r) { - super(i, c); - response = r; - } - - public double getResponse() { - return response; - } - - public void setResponse(double response) { - this.response = response; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Tree.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Tree.java deleted file mode 100644 index 008575993a1..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/Tree.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - - -public class Tree { - - private String id; - private String comment; - - private InternalNode root; - private int nInternalNodes; // number of internal nodes - - - public Tree() {} - - public Tree(String id, String comment) { - this.id = id; - this.comment = comment; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getComment() { - return (comment == null ? "" : comment); - } - - public void setComment(String comment) { - this.comment = comment; - } - - public InternalNode getRoot() { - return root; - } - - public void setRoot(InternalNode root) { - this.root = root; - } - - public int getNumInternalNodes() { - return nInternalNodes; - } - - public void incrInteralNodes() { - nInternalNodes++; - } - - public void setNumInternalNodes(int n) { - nInternalNodes = n; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreeNode.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreeNode.java deleted file mode 100644 index a927e014600..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreeNode.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public class TreeNode { - - private String id; - private String comment; - private int idx; - - public TreeNode(String i, String c) { - id = i; - comment = c; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getComment() { - return (comment == null ? "" : comment); - } - - public void setComment(String comment) { - this.comment = comment; - } - - public int getIndex() { - return idx; - } - - public void setIndex(int idx) { - this.idx = idx; - } - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreeNodeVisitor.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreeNodeVisitor.java deleted file mode 100644 index 8ef4f5d25fd..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreeNodeVisitor.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -public interface TreeNodeVisitor { - - public void visit(TreeNode node); - -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreenetFunction.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreenetFunction.java deleted file mode 100644 index a4569df2629..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/entity/TreenetFunction.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.entity; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.w3c.dom.Attr; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import com.yahoo.yst.libmlr.converter.parser.DecisionTreeXmlException; - -public class TreenetFunction extends MlrFunction { - - private String ns; // namespace - private ArrayList<Tree> treeArylst; - private HashSet<String> featureSet; - private HashSet<String> labelSet; - protected ArrayList<EarlyExit> earlyExitArylst; - - - public TreenetFunction() { - treeArylst = new ArrayList<Tree>(500); - featureSet = new HashSet<String>(); - labelSet = new HashSet<String>(); - earlyExitArylst = new ArrayList<EarlyExit>(5); - } - - public void setFunctionName(String id) { - funcId = id; - functionName = "mlr" + id; - /* - Pattern p = Pattern.compile("[^\\d]+(\\d+)\\w*"); - Matcher m = p.matcher(functionName); - if (!m.matches()) - throw new IllegalArgumentException("not a valid functionName"); - - funcId = m.group(1); - */ - ns = "mlr" + funcId + "ns"; - } - - public String getNameSpace() { - return ns; - } - - public int getNumberOfTrees() { - return treeArylst.size(); - } - - public Tree getTree(int i) { - return treeArylst.get(i); - } - - public void setTree(Tree t) { - treeArylst.add(t); - } - - public HashSet<String> getFeatureSet() { - return featureSet; - } - - public HashSet<String> getLabelSet() { - return labelSet; - } - - public void addFeature(String f) { - featureSet.add(f); - } - - public void addLabel(String lbl) { - if (labelSet.contains(lbl)) - throw new DecisionTreeXmlException("Label " + lbl + " existed."); - labelSet.add(lbl); - } - - public void removeLabelSet() { - labelSet = null; - } - - public void getAllFeatures() { - for (String f: featureSet) { - System.out.println(f); - } - } - - public void addEarlyExit(EarlyExit earx) { - earlyExitArylst.add(earx); - } - - public int getNumEarlyExits() { - return earlyExitArylst.size(); - } - - public EarlyExit getEarlyExit(int i) { - return earlyExitArylst.get(i); - } -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/parser/DecisionTreeXmlException.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/parser/DecisionTreeXmlException.java deleted file mode 100644 index ab264b51d29..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/parser/DecisionTreeXmlException.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.parser; - -public class DecisionTreeXmlException extends RuntimeException { - - public DecisionTreeXmlException() { - super(); - } - - public DecisionTreeXmlException(String msg) { - super(msg); - } - - public DecisionTreeXmlException(String msg, Throwable cause) { - super(msg, cause); - } -} diff --git a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/parser/MlrXmlParser.java b/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/parser/MlrXmlParser.java deleted file mode 100644 index 1c52b5e9309..00000000000 --- a/libmlr/src/main/java/com/yahoo/yst/libmlr/converter/parser/MlrXmlParser.java +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yst.libmlr.converter.parser; - -import com.yahoo.yst.libmlr.converter.XmlUtils; -import com.yahoo.yst.libmlr.converter.entity.*; -import org.w3c.dom.*; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.File; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.logging.Logger; - -/** - * Parses Treenet output V5 into Abstract Treenet XML File format. - * - * @author allenwei - * - */ -public class MlrXmlParser { - - private static Logger logger = Logger.getLogger("com.yahoo.yst.libmlrutil.TnXmlParser"); - private static final String errNormAttr = "<Normalize>: All or none of attributes mean0-3, sd0-3, a0-3, b0-3 are required"; - private static final String errPolyAttr = "<Normalize>: All or none of attributes a0-3 are required"; - - private HashSet<String> treeIdSet = new HashSet<String>(500); - private HashSet<String> nodeIdSet = new HashSet<String>(10000); - private HashSet<String> respIdSet = new HashSet<String>(10000); - - public MlrFunction parseXmlFile(String fileName) throws DecisionTreeXmlException { - - File file = new File(fileName); - if (!file.exists()) { - String errMsg = fileName + " does not exist."; - logErrors(errMsg); - throw new DecisionTreeXmlException(errMsg); - } - - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = null; - - try { - docBuilder = dbf.newDocumentBuilder(); - Document doc = docBuilder.parse(file); - Element eltMlrFnc = doc.getDocumentElement(); - if (!eltMlrFnc.getTagName().equals("MlrFunction")) { - String errMsg = "The top element must be <MlrFunction>"; - logErrors(errMsg); - throw new DecisionTreeXmlException(errMsg); - } - - return parseMlrFunction(eltMlrFnc); - - //System.out.println("features: " + tnFunc.getFeatureSet().size()); - //System.out.println("labels: " + tnFunc.getLabelSet().size()); - - } catch (ParserConfigurationException pe) { - String errMsg = "Cannot construct XML DocumentBuilder"; - logErrors(pe, errMsg); - throw new DecisionTreeXmlException(errMsg, pe); - - } catch (DecisionTreeXmlException te) { - throw te; - - } catch (Exception ex) { - String errMsg ="Errors found parsing XML"; - logErrors(errMsg); - ex.printStackTrace(); - throw new DecisionTreeXmlException(errMsg, ex); - } - } - - private MlrFunction parseMlrFunction(Element eltMlrFnc) { - MlrFunction mlrFunc = null; - - Element eltDecisionTree = getFirstChildElementByName(eltMlrFnc, "DecisionTree", true); - TreenetFunction tnFunc = new TreenetFunction(); - String id = getAttribute(eltMlrFnc, "name", true); - try { - Integer.parseInt(id); - } catch (NumberFormatException nex) { - throw new DecisionTreeXmlException("name in <MlrFunction> should be an integer " + id , nex); - } - tnFunc.setFunctionName(id); - parseDecisionTree(eltDecisionTree, tnFunc); - - mlrFunc = tnFunc; - - if (mlrFunc != null) { - Element eltEpilog = getFirstChildElementByName(eltMlrFnc, "Epilogue", false); - if (eltEpilog != null) { - Epilog epilog = parseEpilog(eltEpilog); - mlrFunc.setEpilog(epilog); - } - } - - return mlrFunc; - } - - private void parseDecisionTree(Element eltDecisionTree, TreenetFunction tnFunc) { - parseForest(getFirstChildElementByName(eltDecisionTree, "Forest", true), tnFunc); - - Element eltEarlyExits = getFirstChildElementByName(eltDecisionTree, "EarlyExits", false); - if (eltEarlyExits != null) - parseEarlyExits(eltEarlyExits, tnFunc); - } - - private void parseForest(Element eltForest, TreenetFunction tnFunc) { - //String strTotal = eltForest.getAttribute("total"); - // tnFunc.setNumTrees(Integer.parseInt(eltForest.getAttribute("total"))); - - ArrayList<Element> nl = XmlUtils.getChildrenByName(eltForest, "Tree"); - int n = nl.size(); - if (n == 0) - throw new DecisionTreeXmlException("<Forest> should have at least one <Tree> element"); - - for (int i = 0; i < n; i++) { - parseTree(nl.get(i), tnFunc); - } - } - - private void parseTree(Element eltTree, TreenetFunction tnFunc) { - String comment = getAttribute(eltTree, "comment", false); - String id = getAttribute(eltTree, "id", true); - if (treeIdSet.contains(id)) - throw new DecisionTreeXmlException("Duplicate tree id " + id); - else - treeIdSet.add(id); - - Tree tr = new Tree(id, comment); - tnFunc.setTree(tr); - - // DEBUG - //System.out.println("tree " + id); - ArrayList<Element> list = XmlUtils.getChildrenByName(eltTree, "Node"); - if (list == null || list.size() != 1) - throw new DecisionTreeXmlException("<Tree> should have exactly one root <Node> element"); - - Element eltNode = list.get(0); - InternalNode root = parseInternalNode(eltNode, tnFunc, tr); - tr.setRoot(root); - } - - private TreeNode parseTreeNode(Element eltNode, TreenetFunction tnFunc, Tree tr) { - String tag = eltNode.getNodeName(); - if (tag.equals("Node")) - return parseInternalNode(eltNode, tnFunc, tr); - else if (tag.equals("Response")) - return parseResponse(eltNode, tnFunc); - else - throw new DecisionTreeXmlException("ERROR: unknown tag <" + tag + ">. Should never reach here."); - } - - private InternalNode parseInternalNode(Element eltNode, TreenetFunction tnFunc, Tree tr) { - tr.incrInteralNodes(); - - String id = getAttribute(eltNode, "id", true); - if (nodeIdSet.contains(id)) - throw new DecisionTreeXmlException("Duplicate Internal Node id " + id); - else - nodeIdSet.add(id); - - String comment = getAttribute(eltNode, "comment", false); - - String feature = getAttribute(eltNode, "feature", true); - tnFunc.addFeature(feature); - - String value = getAttribute(eltNode, "value", true); - try { - Double.parseDouble(value); - } catch (NumberFormatException nfex) { - String errMsg = "Node " + id + ": value not a number: " + value; - throw new DecisionTreeXmlException(errMsg, nfex); - } - - ArrayList<Node> childNodes = new ArrayList<Node>(5); - - NodeList nl = eltNode.getChildNodes(); - int n = nl.getLength(); - Node nd; - for (int i = 0; i < n; i++) { - nd = nl.item(i); - if (nd.getNodeType() == Node.ELEMENT_NODE) { - String tag = nd.getNodeName(); - if (tag.equals("Node") || tag.equals("Response")) { - childNodes.add(nd); - } - } - } - - int numChildNodes = childNodes.size(); - if (numChildNodes != 2) { - String strNode = "Node: id=" + id + " " + feature + " " + value; - String errMsgNodes = "ERROR: A <Node> element should have exactly 2 child nodes. A child node can be <Node> or <Response>. " + strNode; - throw new DecisionTreeXmlException(errMsgNodes); - } - - TreeNode left = parseTreeNode((Element)childNodes.get(0), tnFunc, tr); - TreeNode right = parseTreeNode((Element)childNodes.get(1), tnFunc, tr); - - return new InternalNode(id, comment, feature, value, left, right); - } - - private ResponseNode parseResponse(Element eltResponse, TreenetFunction tnFunc) { - String id = getAttribute(eltResponse, "id", true); - if (respIdSet.contains(id)) - throw new DecisionTreeXmlException("Duplicate Response Node id " + id); - else - respIdSet.add(id); - - String comment = getAttribute(eltResponse, "comment", false); - - String strValue = eltResponse.getAttribute("value"); - double value; - try { - value = Double.parseDouble(strValue); - } catch (NumberFormatException ne) { - throw new DecisionTreeXmlException("Response Node " + id + " does not contain a double value. value=" + strValue); - } - - return new ResponseNode(id, comment, value); - } - - private void parseEarlyExits(Element eltEarlyExits, TreenetFunction tnFunc) { - ArrayList<Element> nl = XmlUtils.getChildrenByName(eltEarlyExits, "Exit"); - if (nl != null) { - int n = nl.size(); - for (int i = 0; i < n; i++) { - parseExit(nl.get(i), tnFunc); - } - } - } - - private void parseExit(Element eltExit, TreenetFunction tnFunc) { - String attr = getAttribute(eltExit, "tree", true); - int tree; - try { - tree = Integer.parseInt(attr); - } catch (NumberFormatException ex) { - String errMsg = "Invalid value for attribute tree: " + attr; - throw new DecisionTreeXmlException(errMsg); - } - - String strValue = getAttribute(eltExit, "value", true); - try { - Double.parseDouble(attr); - } catch (NumberFormatException ex) { - String errMsg = "Invalid value for attribute value: " + attr; - throw new DecisionTreeXmlException(errMsg); - } - - attr = getAttribute(eltExit, "op", true); - Operator op; - try { - op = Operator.parse(attr); - } catch (IllegalArgumentException ex) { - String errMsg = "Invalid value for attribute op: " + attr; - throw new DecisionTreeXmlException(errMsg); - } - - tnFunc.addEarlyExit(new EarlyExit(tree, op, strValue)); - - } - - private Epilog parseEpilog(Element eltEpilog) { - Element eltOp = XmlUtils.getFirstChildElement(eltEpilog); - if (eltOp.getNodeName().equals("Normalize")) { - try { - return parseNormalize(eltOp); - } catch (DecisionTreeXmlException e) { - return null; - } - } else if (eltOp.getNodeName().equals("Polytransform")) { - return parsePolytransform(eltOp); - } - else { - return null; - } - } - - private Epilog parseNormalize(Element eltNorm) { - Epilog epilog = new Epilog(); - FuncNormalize func = new FuncNormalize(); - epilog.setFunction(func); - - String strIsInv = getBoolAttribute(eltNorm, "isInverted", false); - if (strIsInv != null && strIsInv.equals("true")) { - String strInvFrom = getDoubleAttribute(eltNorm, "invertedFrom", true); - func.setInvertMethod(FuncNormalize.INV_INVERSION); - func.setInvertedFrom(strInvFrom); - } - - String strIsNeg = getAttribute(eltNorm, "isNegated", false); - if (strIsNeg != null && strIsNeg.equals("true")) { - if (func.getInvertMethod() == FuncNormalize.INV_NONE) - func.setInvertMethod(FuncNormalize.INV_NEGATION); - else - throw new DecisionTreeXmlException("cannot have both isInverted and isNegated defined in element <Normalize>"); - } - - func.setMean0(getDoubleAttribute(eltNorm, "mean0", false)); - func.setMean1(getDoubleAttribute(eltNorm, "mean1", false)); - func.setMean2(getDoubleAttribute(eltNorm, "mean2", false)); - func.setMean3(getDoubleAttribute(eltNorm, "mean3", false)); - func.setSd0(getDoubleAttribute(eltNorm, "sd0", false)); - func.setSd1(getDoubleAttribute(eltNorm, "sd1", false)); - func.setSd2(getDoubleAttribute(eltNorm, "sd2", false)); - func.setSd3(getDoubleAttribute(eltNorm, "sd3", false)); - func.setA0(getDoubleAttribute(eltNorm, "a0", false)); - func.setA1(getDoubleAttribute(eltNorm, "a1", false)); - func.setA2(getDoubleAttribute(eltNorm, "a2", false)); - func.setA3(getDoubleAttribute(eltNorm, "a3", false)); - func.setB0(getDoubleAttribute(eltNorm, "b0", false)); - func.setB1(getDoubleAttribute(eltNorm, "b1", false)); - func.setB2(getDoubleAttribute(eltNorm, "b2", false)); - func.setB3(getDoubleAttribute(eltNorm, "b3", false)); - - if (!func.validateParams()) - throw new DecisionTreeXmlException(errNormAttr); - - return epilog; - } - - private Epilog parsePolytransform(Element eltOp) { - Epilog epilog = new Epilog(); - FuncPolytransform func = new FuncPolytransform(); - - func.setA0(getDoubleAttribute(eltOp, "a0", false)); - func.setA1(getDoubleAttribute(eltOp, "a1", false)); - func.setA2(getDoubleAttribute(eltOp, "a2", false)); - func.setA3(getDoubleAttribute(eltOp, "a3", false)); - - if (!func.validateParams()) - throw new DecisionTreeXmlException(errPolyAttr); - - epilog.setFunction(func); - return epilog; - } - - /** - * Checks if the attribute name exists. - * - * @param eltNorm - the element containing the attribute - * @param attr - attribute name - * @return true if the attribute exists; or false, otherwise. - */ - private boolean checkAttrExist(Element eltNorm, String attr) { - Attr attrNode = eltNorm.getAttributeNode(attr); - if (attrNode != null) - return true; - else - return false; - } - - /** - * Returns the value of attribute. - * - * @param elt - * @param attr - * @param reqd - * @return If the attribute exists, the value of the attribute is returned, otherwise null is returned. - */ - private String getAttribute(Element elt, String attr, boolean reqd) { - Attr attrNode = elt.getAttributeNode(attr); - String val = null; - if (attrNode != null) - val = elt.getAttribute(attr); - - if (reqd && (val == null || val.equals(""))) - throw new DecisionTreeXmlException(elt.getTagName() + ": missing required attribute " + attr); - return val; - } - - private String getBoolAttribute(Element elt, String attr, boolean reqd) { - String strVal = getAttribute(elt, attr, reqd); - - if (strVal == null || - ((strVal.equals("true") || strVal.equals("false")))) { - return strVal; - } else { - String errMsg = "Attribute " + attr + " in Element " + elt.getTagName() + " is not a valid boolean value: " + strVal; - throw new DecisionTreeXmlException(errMsg); - } - } - - private String getIntAttribute(Element elt, String attr, boolean reqd) { - String strVal = getAttribute(elt, attr, reqd); - try { - if (strVal != null) - Integer.parseInt(strVal); - return strVal; - } catch (NumberFormatException ne) { - String errMsg = "Attribute " + attr + " in Element " + elt.getTagName() + " is not a valid integer: " + strVal; - throw new DecisionTreeXmlException(errMsg); - } - } - - private String getDoubleAttribute(Element elt, String attr, boolean reqd) { - String strVal = getAttribute(elt, attr, reqd); - try { - if (strVal != null) - Double.parseDouble(strVal); - return strVal; - } catch (NumberFormatException ne) { - String errMsg = "Attribute " + attr + " in Element " + elt.getTagName() + " is not a valid double: " + strVal; - throw new DecisionTreeXmlException(errMsg); - } - } - - private Element getFirstChildElementByName(Element parent, String childName, boolean reqd) { - Element elt = XmlUtils.getFirstChildElementByName(parent, childName); - if (elt == null && reqd) - throw new DecisionTreeXmlException(elt.getTagName() + ": missing required element " + childName); - return elt; - } - - private static void logErrors(String msg) { - logger.severe(msg); - System.out.println(msg); - } - - private static void logErrors(Exception ex, String msg) { - String errMsg = ex.getClass().getName() + " " + ex.getMessage() + ": " + msg; - logger.severe(errMsg); - System.out.println(errMsg); - } - - public static void main(String[] args) { - String fileName = "C:\\yst\\libMLR_framework\\mlr3135.xml"; - new MlrXmlParser().parseXmlFile(fileName); - } - -} diff --git a/libmlr/src/main/java/config/header_template.txt b/libmlr/src/main/java/config/header_template.txt deleted file mode 100644 index b37431d84ef..00000000000 --- a/libmlr/src/main/java/config/header_template.txt +++ /dev/null @@ -1,17 +0,0 @@ -/** - * File: {0} - * Package: search/secore/libs/mlr - * Description: - * - * Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - * - * Confidential - * - * Conversion Time: {1,date,long} {1,time,long} - * - * MODEL_SIZE: {2} trees x {3} leaf nodes - */ -#include "mlrfeatures.h" -#include "mlrscorereq.h" -#include "mlrfns.h" - diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index db2ed6b0b83..f2393edeff3 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -14,6 +14,7 @@ import java.util.logging.Logger; * @author Stig Bakken */ public abstract class AbstractPluginLoader implements PluginLoader { + private static final Logger log = Logger.getLogger(AbstractPluginLoader.class.getName()); public abstract void loadPlugins(); diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java index 5c6569d9b9b..22453528777 100644 --- a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -14,6 +14,7 @@ import com.yahoo.logserver.handlers.replicator.ReplicatorPlugin; * @author Stig Bakken */ public class BuiltinPluginLoader extends AbstractPluginLoader { + private static final Logger log = Logger.getLogger(BuiltinPluginLoader.class.getName()); public void loadPlugins() { diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index a62e5c525b1..f1e1665a7d1 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -231,9 +231,7 @@ public class ArchiverHandler extends AbstractLogHandler { private void setRootDir(String rootDir) { // roundabout way of setting things, but this way we can - // get around Java's ineptitude for file handling. (relative - // paths in File are broken) - // + // get around Java's ineptitude for file handling (relative paths in File are broken) absoluteRootDir = new File(rootDir).getAbsolutePath(); root = new File(absoluteRootDir); @@ -242,8 +240,7 @@ public class ArchiverHandler extends AbstractLogHandler { log.log(LogLevel.DEBUG, "Using " + absoluteRootDir + " as root"); } else { if (! root.mkdirs()) { - log.log(LogLevel.ERROR, - "Unable to create directory " + absoluteRootDir); + log.log(LogLevel.ERROR, "Unable to create directory " + absoluteRootDir); } else { log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir); } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java index 58fcfe88465..e3989a22c66 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java @@ -10,7 +10,7 @@ import java.util.Random; * An RPCService represents a set of remote sessions matching a service pattern. The sessions are monitored using the * slobrok. If multiple sessions are available, round robin is used to balance load between them. * - * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> + * @author havardpe */ public class RPCService { diff --git a/metrics/src/main/java/com/yahoo/metrics/Metric.java b/metrics/src/main/java/com/yahoo/metrics/Metric.java index ff86662247a..f1f389a4dfe 100644 --- a/metrics/src/main/java/com/yahoo/metrics/Metric.java +++ b/metrics/src/main/java/com/yahoo/metrics/Metric.java @@ -8,8 +8,8 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.List; -public abstract class Metric -{ +public abstract class Metric { + private String name; private String tags; private String description; @@ -146,7 +146,7 @@ public abstract class Metric writer.openTag(getXMLTag()); - if (getXMLTag() != getName()) { + if ( ! getXMLTag().equals(getName())) { writer.attribute(TAG_NAME, getName()); } @@ -87,7 +87,6 @@ <module>jdisc_jetty</module> <module>jdisc_messagebus_service</module> <module>jrt</module> - <module>libmlr</module> <module>linguistics</module> <module>logd</module> <module>logserver</module> diff --git a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java index a19d59c193d..b74960eb6e9 100644 --- a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java +++ b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java @@ -19,13 +19,13 @@ import com.yahoo.container.StatisticsConfig; * will be disabled by initializing the Statistics class with a null config * object. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class StatisticsImpl extends AbstractComponent implements Statistics { + private final Timer worker; private final StatisticsConfig config; - private static final Logger log = Logger.getLogger(StatisticsImpl.class - .getName()); + private static final Logger log = Logger.getLogger(StatisticsImpl.class.getName()); private final int collectioninterval; private final int logginginterval; // default access for testing only @@ -44,9 +44,9 @@ public final class StatisticsImpl extends AbstractComponent implements Statistic * if logging interval is smaller than collection interval, or * collection interval is not a multiplum of logging interval */ - public StatisticsImpl(final StatisticsConfig config) { - final int l = (int) config.loggingintervalsec(); - final int c = (int) config.collectionintervalsec(); + public StatisticsImpl(StatisticsConfig config) { + int l = (int) config.loggingintervalsec(); + int c = (int) config.collectionintervalsec(); if (l != 0 && l < c) { throw new IllegalArgumentException( @@ -68,15 +68,13 @@ public final class StatisticsImpl extends AbstractComponent implements Statistic * Cancel internal worker thread and do any other necessary cleanup. The * internal worker thread is a daemon thread, so not calling this will not * hamper a clean exit from the VM. - * - * @since 5.1.4 */ @Override public void deconstruct() { worker.cancel(); } - private void schedule(final Handle h) { + private void schedule(Handle h) { if (logginginterval != 0) { h.run(); // We use the rather creative assumption that there is @@ -94,9 +92,9 @@ public final class StatisticsImpl extends AbstractComponent implements Statistic * the internal state of this object. */ @Override - public void register(final Handle h) { + public void register(Handle h) { synchronized (handles) { - final Handle oldHandle = handles.get(h.getName()); + Handle oldHandle = handles.get(h.getName()); if (oldHandle == h) { log.log(Level.WARNING, "Handle [" + h + "] already registered"); return; @@ -115,9 +113,9 @@ public final class StatisticsImpl extends AbstractComponent implements Statistic * Remove a named handler from the set of working handlers. */ @Override - public void remove(final String name) { + public void remove(String name) { synchronized (handles) { - final Handle oldHandle = handles.remove(name); + Handle oldHandle = handles.remove(name); if (oldHandle != null) { oldHandle.cancel(); } @@ -141,7 +139,7 @@ public final class StatisticsImpl extends AbstractComponent implements Statistic @Override public int purge() { synchronized (handles) { - final Iterator<Handle> it = handles.values().iterator(); + Iterator<Handle> it = handles.values().iterator(); while (it.hasNext()) { final Handle h = it.next(); if (h.isCancelled()) { @@ -151,4 +149,5 @@ public final class StatisticsImpl extends AbstractComponent implements Statistic return worker == null ? 0 : worker.purge(); } } + } diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java index d60bad36d47..b1e08300d69 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Value.java +++ b/statistics/src/main/java/com/yahoo/statistics/Value.java @@ -361,8 +361,7 @@ public class Value extends Handle { throw new IllegalStateException("The Limits instance must be frozen."); } if (parameters.histogramId != HistogramType.REGULAR) { - log.log(LogLevel.WARNING, HISTOGRAM_TYPE_WARNING - + " '" + name + "'"); + log.log(LogLevel.WARNING, HISTOGRAM_TYPE_WARNING + " '" + name + "'"); } this.histogramId = HistogramType.REGULAR; this.histogram = parameters.limits; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java index ba89ed550de..670b30f880d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java @@ -14,6 +14,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * Reads an input stream of xml, sends these to session. + * * @author dybis */ public class XmlFeedReader { @@ -23,12 +24,13 @@ public class XmlFeedReader { public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) throws Exception { - SAXParserFactory parserFactor = SAXParserFactory.newInstance(); - parserFactor.setValidating(false); - parserFactor.setNamespaceAware(false); - final SAXParser parser = parserFactor.newSAXParser(); + SAXParserFactory parserFactory = SAXParserFactory.newInstance(); + // XXE prevention: + parserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); + parserFactory.setValidating(false); + parserFactory.setNamespaceAware(false); + SAXParser parser = parserFactory.newSAXParser(); SAXClientFeeder saxClientFeeder = new SAXClientFeeder(feedClient, numSent); - SAXClientFeeder handler = saxClientFeeder; InputSource inputSource = new InputSource(); inputSource.setEncoding(StandardCharsets.UTF_8.displayName()); @@ -36,8 +38,7 @@ public class XmlFeedReader { // This is to send events about CDATA to the saxClientFeeder // (https://docs.oracle.com/javase/tutorial/jaxp/sax/events.html) parser.setProperty("http://xml.org/sax/properties/lexical-handler", saxClientFeeder); - - parser.parse(inputSource, handler); + parser.parse(inputSource, saxClientFeeder); } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index 51138db8d00..76aaf283a16 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -37,6 +37,7 @@ public class CommandLineOptions { public static final String PRIORITY_OPTION = "priority"; public static final String LOADTYPE_OPTION = "loadtype"; public static final String JSONOUTPUT_OPTION = "jsonoutput"; + public static final String XMLOUTPUT_OPTION = "xmloutput"; private final Options options = createOptions(); private final InputStream stdIn; @@ -136,6 +137,11 @@ public class CommandLineOptions { .desc("JSON output") .longOpt(JSONOUTPUT_OPTION).build()); + options.addOption(Option.builder("x") + .hasArg(false) + .desc("XML output (default format)") + .longOpt(XMLOUTPUT_OPTION).build()); + return options; } @@ -165,11 +171,16 @@ public class CommandLineOptions { boolean noRetry = cl.hasOption(NORETRY_OPTION); boolean showDocSize = cl.hasOption(SHOWDOCSIZE_OPTION); boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION); + boolean xmlOutput = cl.hasOption(XMLOUTPUT_OPTION); int trace = getTrace(cl); DocumentProtocol.Priority priority = getPriority(cl); double timeout = getTimeout(cl); Iterator<String> documentIds = getDocumentIds(cl); + if (jsonOutput && xmlOutput) { + throw new IllegalArgumentException("Cannot combine both xml and json output"); + } + if (printIdsOnly && headersOnly) { throw new IllegalArgumentException("Print ids and headers only options are mutually exclusive."); } @@ -216,7 +227,7 @@ public class CommandLineOptions { .setTraceLevel(trace) .setPriority(priority) .setTimeout(timeout) - .setJsonOutput(jsonOutput) + .setJsonOutput(!jsonOutput && !xmlOutput ? false : jsonOutput) // TODO Vespa 7 Change default to JSON .build(); } catch (ParseException pe) { throw new IllegalArgumentException(pe.getMessage()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index 16d860a1d98..1df9a1e38cd 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -349,6 +349,13 @@ public class VdsVisit { .hasArg(false) .build()); + options.addOption(Option.builder("x") + .longOpt("xmloutput") + .desc("Output documents as XML (default format)") + .hasArg(false) + .build()); + + options.addOption(Option.builder() .longOpt("bucketspace") .hasArg(true) @@ -573,9 +580,13 @@ public class VdsVisit { throttlePolicy.setMaxPendingCount(((Number)line.getParsedOptionValue("maxpendingsuperbuckets")).intValue()); params.setThrottlePolicy(throttlePolicy); } - if (line.hasOption("jsonoutput")) { - allParams.setJsonOutput(true); + + boolean jsonOutput = line.hasOption("jsonoutput"); + boolean xmlOutput = line.hasOption("xmloutput"); + if (jsonOutput && xmlOutput) { + throw new IllegalArgumentException("Cannot combine both xml and json output"); } + allParams.setJsonOutput(!jsonOutput && !xmlOutput ? false : jsonOutput); // TODO Vespa 7 Change default to JSON allParams.setVisitorParameters(params); return allParams; diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java index 9452a2924ba..cceac7e84bb 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java +++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java @@ -24,7 +24,8 @@ public class ProcessExecuter { public Pair<Integer, String> exec(String command) throws IOException { StringTokenizer tok = new StringTokenizer(command); List<String> tokens = new ArrayList<>(); - while (tok.hasMoreElements()) tokens.add(tok.nextToken()); + while (tok.hasMoreElements()) + tokens.add(tok.nextToken()); return exec(tokens.toArray(new String[0])); } @@ -43,10 +44,10 @@ public class ProcessExecuter { InputStream is = p.getInputStream(); while (true) { int b = is.read(); - if (b==-1) break; + if (b == -1) break; ret.append((char)b); } - int rc=0; + int rc = 0; try { rc = p.waitFor(); } catch (InterruptedException e) { diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java index f4cd355b0e1..35fc41ab988 100644 --- a/vespajlib/src/main/java/com/yahoo/text/XML.java +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; +import java.io.File; import java.io.IOException; import java.io.Reader; import java.io.StringReader; @@ -397,18 +398,43 @@ public class XML { } /** - * Returns the Document of an XML file reader + * Returns the parsed Document from an XML file * - * @throws RuntimeException if the root Document cannot be returned + * @throws IllegalArgumentException if the file cannot be opened or parsed + */ + public static Document getDocument(File xmlFile) { + try { + return getDocumentBuilder().parse(xmlFile); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read '" + xmlFile + "'", e); + } + catch (SAXParseException e) { + throw new IllegalArgumentException("Could not parse '" + xmlFile + + "', error at line " + e.getLineNumber() + ", column " + e.getColumnNumber(), e); + } + catch (SAXException e) { + throw new IllegalArgumentException("Could not parse '" + xmlFile + "'", e); + } + } + + /** + * Returns the parsed Document from an XML file + * + * @throws IllegalArgumentException if the file cannot be opened or parsed */ public static Document getDocument(Reader reader) { try { return getDocumentBuilder().parse(new InputSource(reader)); - } catch (IOException e) { + } + catch (IOException e) { throw new IllegalArgumentException("Could not read '" + reader + "'", e); - } catch (SAXParseException e) { - throw new IllegalArgumentException("Could not parse '" + reader + "', error at line " + e.getLineNumber() + ", column " + e.getColumnNumber(), e); - } catch (SAXException e) { + } + catch (SAXParseException e) { + throw new IllegalArgumentException("Could not parse '" + reader + + "', error at line " + e.getLineNumber() + ", column " + e.getColumnNumber(), e); + } + catch (SAXException e) { throw new IllegalArgumentException("Could not parse '" + reader + "'", e); } } @@ -416,16 +442,15 @@ public class XML { /** * Returns the Document of the string XML payload */ - public static Document getDocument(String string) { - return getDocument(new StringReader(string)); + public static Document getDocument(String xmlString) { + return getDocument(new StringReader(xmlString)); } /** * Creates a new XML DocumentBuilder * * @return a DocumentBuilder - * @throws RuntimeException - * if we fail to create one + * @throws RuntimeException if we fail to create one */ public static DocumentBuilder getDocumentBuilder() { return getDocumentBuilder("com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl", null); @@ -434,13 +459,9 @@ public class XML { /** * Creates a new XML DocumentBuilder * - * @param implementation - * which jaxp implementation should be used - * @param classLoader - * which class loader should be used when getting a new - * DocumentBuilder - * @throws RuntimeException - * if we fail to create one + * @param implementation which jaxp implementation should be used + * @param classLoader which class loader should be used when getting a new DocumentBuilder + * @throws RuntimeException if we fail to create one * @return a DocumentBuilder */ public static DocumentBuilder getDocumentBuilder(String implementation, ClassLoader classLoader) { @@ -448,17 +469,18 @@ public class XML { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(implementation, classLoader); factory.setNamespaceAware(true); factory.setXIncludeAware(true); + // Prevent XXE + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); return factory.newDocumentBuilder(); } catch (ParserConfigurationException e) { - throw new RuntimeException("Could not create an XML builder"); + throw new RuntimeException("Could not create an XML builder", e); } } /** * Returns the child Element objects from a w3c dom spec * - * @return List of elements. Empty list (never null) if none found or if the - * given element is null + * @return List of elements. Empty list (never null) if none found or if the given element is null */ public static List<Element> getChildren(Element spec) { List<Element> children = new ArrayList<>(); @@ -479,8 +501,7 @@ public class XML { /** * Returns the child Element objects with given name from a w3c dom spec * - * @return List of elements. Empty list (never null) if none found or the - * given element is null + * @return List of elements. Empty list (never null) if none found or the given element is null */ public static List<Element> getChildren(Element spec, String name) { List<Element> ret = new ArrayList<>(); |