diff options
| -rw-r--r-- | vespaclient-core/pom.xml | 2 | ||||
| -rw-r--r-- | vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java | 22 | ||||
| -rwxr-xr-x | vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java | 31 | ||||
| -rw-r--r-- | vespaclient-core/src/main/java/com/yahoo/feedhandler/InputStreamRequest.java (renamed from vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java) | 14 | ||||
| -rw-r--r-- | vespaclient-core/src/main/java/com/yahoo/feedhandler/ParameterParser.java | 71 | ||||
| -rwxr-xr-x | vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java | 12 | ||||
| -rwxr-xr-x | vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java | 6 | ||||
| -rw-r--r-- | vespaclient-java/pom.xml | 40 | ||||
| -rwxr-xr-x | vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java | 2 | ||||
| -rwxr-xr-x | vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java | 5 | ||||
| -rw-r--r-- | vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java | 1 |
11 files changed, 110 insertions, 96 deletions
diff --git a/vespaclient-core/pom.xml b/vespaclient-core/pom.xml index 5afc2b57868..87f27248b80 100644 --- a/vespaclient-core/pom.xml +++ b/vespaclient-core/pom.xml @@ -19,7 +19,7 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dev</artifactId> + <artifactId>documentapi</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index ac1ed248ee9..f862451c095 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -3,9 +3,9 @@ package com.yahoo.feedapi; import com.yahoo.concurrent.SystemTimer; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.feedhandler.InputStreamRequest; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; import com.yahoo.vespaclient.config.FeederConfig; @@ -26,7 +26,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private Route defaultRoute = null; private long defaultTimeoutMillis = 0; private boolean retryEnabled = true; - private String defaultDocprocChain = null; private boolean defaultAbortOnDocumentError = true; private boolean defaultAbortOnSendError = true; private boolean configChanged = false; @@ -39,7 +38,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib defaultRoute = Route.parse(routeOverride); } - public PropertySetter buildPropertySetter(HttpRequest request) { + public PropertySetter buildPropertySetter(InputStreamRequest request) { String routeParam = null; double timeoutParam = -1; String priorityParam = null; @@ -129,7 +128,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib log.log(Level.FINE, "Received new config (" + "route: " + (defaultRoute != null ? defaultRoute : "<none>") + ", timeout: " + defaultTimeoutMillis + " ms, retry enabled: " + retryEnabled + - ", docproc chain: " + (defaultDocprocChain != null ? defaultDocprocChain : "<none>") + ", abort on doc error: " + defaultAbortOnDocumentError + ", abort on feed error: " + defaultAbortOnSendError + ")"); } @@ -139,16 +137,16 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib /** Route either set by configuration or by explicit request override. May be null */ private Route route; /** Timeout (in milliseconds) */ - private long timeout; - private long totalTimeout; - private long startTime; + private final long timeout; + private final long totalTimeout; + private final long startTime; /** Explicit priority set. May be null */ private DocumentProtocol.Priority priority; - private boolean retryEnabled; - private boolean abortOnDocumentError; - private boolean abortOnFeedError; - private boolean createIfNonExistent; - private int traceLevel; + private final boolean retryEnabled; + private final boolean abortOnDocumentError; + private final boolean abortOnFeedError; + private final boolean createIfNonExistent; + private final int traceLevel; PropertySetter(Route route, long timeout, long totalTimeout, DocumentProtocol.Priority priority, boolean retryEnabled, boolean abortOnDocumentError, boolean abortOnFeedError, diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java index dd95e1dc202..48ba8f83d3c 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java @@ -11,7 +11,6 @@ import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; -import com.yahoo.search.result.ErrorMessage; import java.util.ArrayList; import java.util.List; @@ -21,7 +20,6 @@ import java.util.stream.Stream; public final class FeedResponse implements SharedSender.ResultCallback { private final static Logger log = Logger.getLogger(FeedResponse.class.getName()); - private final List<ErrorMessage> errorMessages = new ArrayList<>(); private final List<String> errors = new ArrayList<>(); private final StringBuilder traces = new StringBuilder(); private final RouteMetricSet metrics; @@ -75,7 +73,7 @@ public final class FeedResponse implements SharedSender.ResultCallback { String str = out.toString(); log.finest(str); - addError(convertErrorCode(err.getCode()), str); + addError(err); } if (abortOnError) { isAborted = true; @@ -92,14 +90,12 @@ public final class FeedResponse implements SharedSender.ResultCallback { } FeedResponse addXMLParseError(String error) { - errorMessages.add(ErrorMessage.createBadRequest(error)); errors.add(error); return this; } - public FeedResponse addError(com.yahoo.container.protect.Error code, String error) { - errorMessages.add(new ErrorMessage(code.code, error)); - errors.add(error); + public FeedResponse addError(Error error) { + errors.add(error.toString()); return this; } @@ -111,27 +107,6 @@ public final class FeedResponse implements SharedSender.ResultCallback { return errors.anyMatch(e -> e.getCode() != DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED); } - private static com.yahoo.container.protect.Error convertErrorCode(int error) { - // We should try to enumerate these error a bit finer. - // Like busy, no space etc. - if (error == DocumentProtocol.ERROR_NO_SPACE) { - return com.yahoo.container.protect.Error.INSUFFICIENT_STORAGE; - } else if (isTransientError(error)) { - return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; - } if (isFatalError(error)) { - return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; - } - return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; - } - - private static boolean isFatalError(int error) { - return (error >= ErrorCode.FATAL_ERROR) && (error < ErrorCode.ERROR_LIMIT); - } - - private static boolean isTransientError(int error) { - return (error >= ErrorCode.TRANSIENT_ERROR) && (error < ErrorCode.FATAL_ERROR); - } - public boolean isSuccess() { return errors.isEmpty(); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/InputStreamRequest.java index e3c61dd26e9..a62ecbd55ec 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/InputStreamRequest.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/InputStreamRequest.java @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespafeeder; - -import com.yahoo.container.jdisc.HttpRequest; +package com.yahoo.feedhandler; import java.io.InputStream; import java.util.HashMap; @@ -16,10 +14,10 @@ import java.util.Map; */ public class InputStreamRequest { - private InputStream input; - private Map<String, String> properties = new HashMap<>(); + private final InputStream input; + private final Map<String, String> properties = new HashMap<>(); - protected InputStreamRequest(InputStream input) { + public InputStreamRequest(InputStream input) { this.input = input; } @@ -31,8 +29,6 @@ public class InputStreamRequest { return properties.get(key); } - public HttpRequest toRequest() { - return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.POST, input, properties); - } + InputStream getData() { return input; } } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/ParameterParser.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ParameterParser.java new file mode 100644 index 00000000000..c907c5a72d8 --- /dev/null +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ParameterParser.java @@ -0,0 +1,71 @@ +package com.yahoo.feedhandler; + +public class ParameterParser { + + /** + * Tries to return the given object as a Long. If it is a Number, treat it + * as a number of seconds, i.e. get a Long representation and multiply by + * 1000. If it has a String representation, try to parse this as a floating + * point number, followed by by an optional unit (seconds and an SI prefix, + * a couple of valid examples are "s" and "ms". Only a very small subset of + * SI prefixes are supported). If no unit is given, seconds are assumed. + * + * @param value some representation of a number of seconds + * @param defaultValue returned if value is null + * @return value as a number of milliseconds + * @throws NumberFormatException if value is not a Number instance and its String + * representation cannot be parsed as a number followed optionally by time unit + */ + public static Long asMilliSeconds(Object value, Long defaultValue) { + if (value == null) return defaultValue; + if (value instanceof Number) return ((Number)value).longValue() * 1000L; + return parseTime(value.toString()); + } + + private static Long parseTime(String time) throws NumberFormatException { + time = time.trim(); + try { + int unitOffset = findUnitOffset(time); + double measure = Double.valueOf(time.substring(0, unitOffset)); + double multiplier = parseUnit(time.substring(unitOffset)); + return (long) (measure * multiplier); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Error parsing '" + time + "'", e); + } + } + + private static int findUnitOffset(String time) { + int unitOffset = 0; + while (unitOffset < time.length()) { + char c = time.charAt(unitOffset); + if (c == '.' || (c >= '0' && c <= '9')) { + unitOffset += 1; + } else { + break; + } + } + if (unitOffset == 0) { + throw new IllegalArgumentException("Invalid number '" + time + "'"); + } + return unitOffset; + } + + private static double parseUnit(String unit) { + unit = unit.trim(); + final double multiplier; + if ("ks".equals(unit)) { + multiplier = 1e6d; + } else if ("s".equals(unit)) { + multiplier = 1000.0d; + } else if ("ms".equals(unit)) { + multiplier = 1.0d; + } else if ("\u00B5s".equals(unit)) { + // microseconds + multiplier = 1e-3d; + } else { + multiplier = 1000.0d; + } + return multiplier; + } + +} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index 06f66c83e1b..75a8c8bd811 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -2,8 +2,6 @@ package com.yahoo.feedhandler; import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.protect.Error; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.Feeder; import com.yahoo.feedapi.JsonFeeder; @@ -11,6 +9,8 @@ import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.feedapi.SimpleFeedAccess; import com.yahoo.feedapi.SingleSender; import com.yahoo.feedapi.XMLFeeder; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; import java.util.List; @@ -33,7 +33,7 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { return new VespaFeedHandler(context); } - public FeedResponse handle(HttpRequest request, RouteMetricSet.ProgressCallback callback, int numThreads) { + public FeedResponse handle(InputStreamRequest request, RouteMetricSet.ProgressCallback callback, int numThreads) { MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); String route = properties.getRoute().toString(); @@ -57,15 +57,15 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { long millis = getTimeoutMillis(request); boolean completed = sender.waitForPending(millis); if (!completed) { - response.addError(Error.TIMEOUT, "Timed out after " + millis + " ms waiting for responses"); + response.addError(new Error(ErrorCode.TIMEOUT, "Timed out after " + millis + " ms waiting for responses")); } response.done(); return response; } - private Feeder createFeeder(SimpleFeedAccess sender, HttpRequest request) { - if ( ! Boolean.valueOf(request.getProperty(JSON_INPUT))) { + private Feeder createFeeder(SimpleFeedAccess sender, InputStreamRequest request) { + if ( ! Boolean.parseBoolean(request.getProperty(JSON_INPUT))) { return new XMLFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); } return new JsonFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java index 0ef0bf7a3ec..e0688520a70 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedhandler; -import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.feedapi.SharedSender; -import com.yahoo.search.query.ParameterParser; import java.io.InputStream; @@ -39,7 +37,7 @@ public abstract class VespaFeedHandlerBase { * original data stream. * @throws IllegalArgumentException if GZIP stream creation failed */ - InputStream getRequestInputStream(HttpRequest request) { + InputStream getRequestInputStream(InputStreamRequest request) { return request.getData(); } @@ -47,7 +45,7 @@ public abstract class VespaFeedHandlerBase { return context.getDocumentTypeManager(); } - protected long getTimeoutMillis(HttpRequest request) { + protected long getTimeoutMillis(InputStreamRequest request) { return ParameterParser.asMilliSeconds(request.getProperty("timeout"), defaultTimeoutMillis); } diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index 230ddc5c7d3..6730f8b26d7 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -13,37 +13,14 @@ <version>8-SNAPSHOT</version> <dependencies> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespaclient-core</artifactId> <version>${project.version}</version> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>defaults</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-core</artifactId> - <scope>compile</scope> - <version>${project.version}</version> - </dependency> - <dependency> - <!-- TODO: remove dependency on container-dev, and instead depend directly on what this module needs! --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dev</artifactId> + <artifactId>vespalog</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -51,16 +28,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>predicate-search-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespajlib</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> @@ -69,6 +36,11 @@ <artifactId>junit-jupiter</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java index 9d03a186ed8..47828915444 100755 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/FileRequest.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespafeeder; +import com.yahoo.feedhandler.InputStreamRequest; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java index 1208a18205f..188ca4321df 100755 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedhandler.FeedResponse; +import com.yahoo.feedhandler.InputStreamRequest; import com.yahoo.feedhandler.VespaFeedHandler; import com.yahoo.log.LogSetup; import com.yahoo.concurrent.SystemTimer; @@ -78,7 +79,7 @@ public class VespaFeeder { if (args.getFiles().isEmpty()) { InputStreamRequest req = new InputStreamRequest(input); setProperties(req, input); - FeedResponse response = handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); + FeedResponse response = handler.handle(req, createProgressCallback(output), args.getNumThreads()); if ( ! response.isSuccess()) { throw renderErrors(response.getErrorList()); } @@ -96,7 +97,7 @@ public class VespaFeeder { final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName)); setProperties(req, inputSnooper); inputSnooper.close(); - FeedResponse response = handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); + FeedResponse response = handler.handle(req, createProgressCallback(output), args.getNumThreads()); if (!response.isSuccess()) { throw renderErrors(response.getErrorList()); } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java index a26672c5f26..80782a82c4c 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java @@ -18,6 +18,7 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedhandler.InputStreamRequest; import com.yahoo.feedhandler.VespaFeedHandler; import com.yahoo.text.Utf8; import com.yahoo.vespaclient.config.FeederConfig; |