Merge pull request #4908 from vespa-engine/balder/improve-error-codes-during-feeding

Balder/improve error codes during feeding

10 files changed, 53 insertions, 20 deletions

diff --git a/bootstrap.sh b/bootstrap.sh
index b71af73c529..29e18226ca7 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -26,7 +26,7 @@ else
 fi
 
 mvn_install() {
-    mvn --quiet --batch-mode --threads 1.5C --no-snapshot-updates install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@"
+    mvn -e -X --quiet --batch-mode --threads 1.5C --no-snapshot-updates install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@"
 }
 
 # Generate vtag map
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java
index ecb36ade5eb..5fa1a57569b 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java
@@ -6,6 +6,7 @@ import static com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR;
 import static com.yahoo.container.protect.Error.BAD_REQUEST;
 import static com.yahoo.container.protect.Error.FORBIDDEN;
 import static com.yahoo.container.protect.Error.ILLEGAL_QUERY;
+import static com.yahoo.container.protect.Error.INSUFFICIENT_STORAGE;
 import static com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR;
 import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER;
 import static com.yahoo.container.protect.Error.NOT_FOUND;
@@ -13,7 +14,6 @@ import static com.yahoo.container.protect.Error.NO_BACKENDS_IN_SERVICE;
 import static com.yahoo.container.protect.Error.TIMEOUT;
 import static com.yahoo.container.protect.Error.UNAUTHORIZED;
 
-import java.net.URLDecoder;
 import java.util.Iterator;
 
 import com.yahoo.collections.Tuple2;
@@ -159,6 +159,8 @@ public final class VespaHeaders {
             return new Tuple2<>(true, Response.Status.BAD_REQUEST);
         if (error.getCode() == INTERNAL_SERVER_ERROR.code)
             return new Tuple2<>(true, Response.Status.INTERNAL_SERVER_ERROR);
+        if (error.getCode() == INSUFFICIENT_STORAGE.code)
+            return new Tuple2<>(true, Response.Status.INSUFFICIENT_STORAGE);
         return NO_MATCH;
     }
 
diff --git a/container-core/src/main/java/com/yahoo/container/protect/Error.java b/container-core/src/main/java/com/yahoo/container/protect/Error.java
index 46b49b1623a..b39a33b1346 100644
--- a/container-core/src/main/java/com/yahoo/container/protect/Error.java
+++ b/container-core/src/main/java/com/yahoo/container/protect/Error.java
@@ -26,7 +26,8 @@ public enum Error {
     FORBIDDEN(15),
     NOT_FOUND(16),
     BAD_REQUEST(17),
-    INTERNAL_SERVER_ERROR(18);
+    INTERNAL_SERVER_ERROR(18),
+    INSUFFICIENT_STORAGE(19);
 
     public final int code;
 
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java
index 57e7070296f..2af36d74b93 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java
@@ -9,14 +9,22 @@ import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.document.DocumentTypeManagerConfigurer;
 import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
 import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet;
-import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.ErrorCode;
+import com.yahoo.messagebus.Protocol;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.Routable;
 import com.yahoo.messagebus.metrics.MetricSet;
 import com.yahoo.messagebus.routing.RoutingContext;
 import com.yahoo.messagebus.routing.RoutingNodeIterator;
 import com.yahoo.messagebus.routing.RoutingPolicy;
 import com.yahoo.text.Utf8String;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
 import java.util.logging.Logger;
 
 /**
@@ -207,7 +215,7 @@ public class DocumentProtocol implements Protocol {
      * NORMAL categories. Traffic in the HIGH end will be usually be prioritized over important maintenance operations.
      * Traffic in the LOW end will be prioritized after these operations.</p>
      */
-    public static enum Priority {
+    public enum Priority {
         HIGHEST(0),
         VERY_HIGH(1),
         HIGH_1(2),
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index aa7688570ff..41f22f22c80 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -4,8 +4,8 @@
 # For a complete reference, please see the online documentation at https://docs.vagrantup.com.
 Vagrant.configure("2") do |config|
 
-  config.vm.box = "TODO"
-  config.vm.box_url = "TODO"
+  config.vm.box = "centos7-desktop"
+  config.vm.box_url = "/Users/balder/vagrant/centos7-desktop-17.0915.1.box"
 
   config.ssh.forward_agent = true
 
@@ -16,8 +16,8 @@ Vagrant.configure("2") do |config|
     vb.gui = true
     vb.name = "vespa-dev"
 
-    vb.memory = "8192"
-    vb.cpus = 4
+    vb.memory = "10240"
+    vb.cpus = 8
   end
   
   # Install required and nice-to-have packages
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
index c24398388d5..36ab8090e95 100755
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java
@@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ClusterListConfig;
 import com.yahoo.cloud.config.SlobroksConfig;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.protect.Error;
 import com.yahoo.document.DocumentId;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.feedapi.FeedContext;
@@ -78,7 +79,7 @@ public class VespaFeedHandlerRemove extends VespaFeedHandlerBase {
         long millis = getTimeoutMillis(request);
         boolean completed = sender.waitForPending(millis);
         if ( ! completed)
-            response.addError("Timed out after "+millis+" ms waiting for responses");
+            response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses");
         return response;
     }
 
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
index 2f0526a21ef..04ca6798b4c 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java
@@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ClusterListConfig;
 import com.yahoo.cloud.config.SlobroksConfig;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.protect.Error;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage;
 import com.yahoo.feedapi.FeedContext;
@@ -78,7 +79,7 @@ public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase {
         long millis = getTimeoutMillis(request);
         boolean completed = sender.waitForPending(millis);
         if ( ! completed)
-            response.addError("Timed out after "+millis+" ms waiting for responses");
+            response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses");
         return response;
     }
 
diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java
index 0380fa2a7c4..b9aeb2e6e69 100755
--- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java
+++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java
@@ -4,11 +4,12 @@ package com.yahoo.feedapi;
 import com.yahoo.concurrent.SystemTimer;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.log.LogLevel;
-import com.yahoo.messagebus.*;
+import com.yahoo.messagebus.EmptyReply;
+import com.yahoo.messagebus.Message;
+import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.ReplyHandler;
 import com.yahoo.clientmetrics.RouteMetricSet;
 
-import java.util.HashMap;
-import java.util.Map;
 import java.util.logging.Logger;
 
 /**
@@ -157,14 +158,14 @@ public class SharedSender implements ReplyHandler {
                 log.log(LogLevel.SPAM, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val());
             }
             if (owner.isAborted()) {
-                log.log(LogLevel.WARNING, "Received reply for file " + owner.toString() + " which is aborted");
+                log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which is aborted");
                 owner.getPending().clear();
                 return;
             }
             if (owner.handleReply(r)) {
                 owner.getPending().dec();
             } else {
-                log.log(LogLevel.WARNING, "Received reply for file " + owner.toString() + " which wants to abort");
+                log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which wants to abort");
                 owner.getPending().clear();
             }
         } else {
diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java
index c235fdfce72..635659204e8 100755
--- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java
+++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java
@@ -9,8 +9,8 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
 import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
 import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
 import com.yahoo.feedapi.SharedSender;
-import com.yahoo.log.LogLevel;
 import com.yahoo.messagebus.Error;
+import com.yahoo.messagebus.ErrorCode;
 import com.yahoo.messagebus.Message;
 import com.yahoo.messagebus.Reply;
 import com.yahoo.search.result.ErrorMessage;
@@ -122,7 +122,7 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res
 
                 String str = out.toString();
                 log.finest(str);
-                addError(str);
+                addError(convertErrorCode(err.getCode()), str);
             }
             if (abortOnError) {
                 isAborted = true;
@@ -149,6 +149,11 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res
         errors.add(error);
         return this;
     }
+    public FeedResponse addError(com.yahoo.container.protect.Error code, String error) {
+        errorMessages.add(new ErrorMessage(code.code, error));
+        errors.add(error);
+        return this;
+    }
 
     public List<String> getErrorList() {
         return errors;
@@ -162,6 +167,19 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res
         return errors.anyMatch(e -> e.getCode() != DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED);
     }
 
+    private static com.yahoo.container.protect.Error convertErrorCode(int error) {
+        // We should try to enumerate these error a bit finer.
+        // Like busy, no space etc.
+        if (error == DocumentProtocol.ERROR_NO_SPACE) {
+            return com.yahoo.container.protect.Error.INSUFFICIENT_STORAGE;
+        } else if (error >= ErrorCode.TRANSIENT_ERROR && (error < ErrorCode.FATAL_ERROR)) {
+            return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR;
+        } if (error >= ErrorCode.FATAL_ERROR && (error < ErrorCode.ERROR_LIMIT)) {
+            return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR;
+        }
+        return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR;
+    }
+
     public boolean isSuccess() {
         return errors.isEmpty();
     }
diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java
index 4b7d913c0f4..8180bfd84ea 100755
--- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java
+++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java
@@ -8,6 +8,7 @@ import com.yahoo.cloud.config.SlobroksConfig;
 import com.yahoo.container.jdisc.EmptyResponse;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.protect.Error;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.feedapi.DocprocMessageProcessor;
 import com.yahoo.feedapi.FeedContext;
@@ -106,7 +107,7 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase {
             long millis = getTimeoutMillis(request);
             boolean completed = sender.waitForPending(millis);
             if (!completed) {
-                response.addError("Timed out after " + millis + " ms waiting for responses");
+                response.addError(Error.TIMEOUT, "Timed out after " + millis + " ms waiting for responses");
             }
             response.done();
             return response;