Return 409 with error code LOAD_BALANCER_NOT_READY if deployment fails due to that

Move LoadBalancerServiceException to config-provisioning so that it can be used from
config server as well.
Return 409 and LOAD_BALANCER_NOT_READY as error code if we get a LoadBalancerServiceException.
Retry deploy on LOAD_BALANCER_NOT_READY.

8 files changed, 56 insertions, 6 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/exception/LoadBalancerServiceException.java
index e5ab519ab94..41aa241621d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceException.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/exception/LoadBalancerServiceException.java
@@ -1,10 +1,10 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.lb;
+package com.yahoo.config.provision.exception;
 
 import com.yahoo.config.provision.TransientException;
 
 /**
- * Transient exception thrown on behalf of a {@link LoadBalancerService}.
+ * Transient exception thrown on behalf of a load balancer service
  *
  * @author mpolden
  */
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/exception/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/exception/package-info.java
new file mode 100644
index 00000000000..5730f3fdb6b
--- /dev/null
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/exception/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.provision.exception;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
index 3d2ecd4a2ca..6559292645c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java
@@ -49,7 +49,8 @@ public class HttpErrorResponse extends HttpResponse {
         REQUEST_TIMEOUT,
         UNKNOWN_VESPA_VERSION,
         PARENT_HOST_NOT_READY,
-        CERTIFICATE_NOT_READY
+        CERTIFICATE_NOT_READY,
+        LOAD_BALANCER_NOT_READY
     }
 
     public static HttpErrorResponse notFoundError(String msg) {
@@ -100,6 +101,10 @@ public class HttpErrorResponse extends HttpResponse {
         return new HttpErrorResponse(CONFLICT, errorCodes.CERTIFICATE_NOT_READY.name(), msg);
     }
 
+    public static HttpErrorResponse loadBalancerNotReady(String msg) {
+        return new HttpErrorResponse(CONFLICT, errorCodes.LOAD_BALANCER_NOT_READY.name(), msg);
+    }
+
     @Override
     public void render(OutputStream stream) throws IOException {
         new JsonFormat(true).encode(stream, slime);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
index 20ee77be9fe..7cdb8d3a60c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.http;
 import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.CertificateNotReadyException;
 import com.yahoo.config.provision.ParentHostUnavailableException;
+import com.yahoo.config.provision.exception.LoadBalancerServiceException;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
@@ -67,6 +68,8 @@ public class HttpHandler extends LoggingRequestHandler {
             return HttpErrorResponse.parentHostNotReady(getMessage(e, request));
         } catch (CertificateNotReadyException e) {
             return HttpErrorResponse.certificateNotReady(getMessage(e, request));
+        } catch (LoadBalancerServiceException e) {
+            return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request));
         } catch (Exception e) {
             log.log(LogLevel.WARNING, "Unexpected exception handling a config server request", e);
             return HttpErrorResponse.internalServerError(getMessage(e, request));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index 651dde375ee..e219c516f95 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -9,14 +9,22 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.CertificateNotReadyException;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.exception.LoadBalancerServiceException;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
+import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.config.server.MockReloadHandler;
 import com.yahoo.vespa.config.server.MockSecretStore;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
@@ -45,6 +53,7 @@ import java.io.File;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -294,6 +303,13 @@ public class SessionPreparerTest {
         prepare(new File("src/test/resources/deploy/hosted-app"), params);
     }
 
+    @Test(expected = LoadBalancerServiceException.class)
+    public void require_that_conflict_is_returned_when_creating_load_balancer_fails() throws IOException {
+        preparer = createPreparer(HostProvisionerProvider.withProvisioner(new FailWithTransientExceptionProvisioner()));
+        var params = new PrepareParams.Builder().applicationId(applicationId("test")).build();
+        prepare(new File("src/test/resources/deploy/hosted-app"), params);
+    }
+
     private void prepare(File app) throws IOException {
         prepare(app, new PrepareParams.Builder().build());
     }
@@ -327,4 +343,22 @@ public class SessionPreparerTest {
                                   ApplicationName.from(applicationName), InstanceName.defaultName());
     }
 
+    private static class FailWithTransientExceptionProvisioner implements Provisioner {
+
+        @Override
+        public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+            throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception"));
+        }
+
+        @Override
+        public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { }
+
+        @Override
+        public void remove(NestedTransaction transaction, ApplicationId application) { }
+
+        @Override
+        public void restart(ApplicationId application, HostFilter filter) { }
+
+    }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java
index 20599e92aa9..780969206a2 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java
@@ -38,7 +38,8 @@ public class ConfigServerException extends RuntimeException {
         REQUEST_TIMEOUT,
         UNKNOWN_VESPA_VERSION,
         PARENT_HOST_NOT_READY,
-        CERTIFICATE_NOT_READY
+        CERTIFICATE_NOT_READY,
+        LOAD_BALANCER_NOT_READY
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index d2942d77b7a..7ee834844af 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -62,6 +62,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Con
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.BAD_REQUEST;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.CERTIFICATE_NOT_READY;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE;
+import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.LOAD_BALANCER_NOT_READY;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active;
@@ -232,7 +233,8 @@ public class InternalStepRunner implements StepRunner {
                 || e.getErrorCode() == ACTIVATION_CONFLICT
                 || e.getErrorCode() == APPLICATION_LOCK_FAILURE
                 || e.getErrorCode() == PARENT_HOST_NOT_READY
-                || e.getErrorCode() == CERTIFICATE_NOT_READY) {
+                || e.getErrorCode() == CERTIFICATE_NOT_READY
+                || e.getErrorCode() == LOAD_BALANCER_NOT_READY) {
                 logger.log("Will retry, because of '" + e.getErrorCode() + "' deploying:\n" + e.getMessage());
                 return Optional.empty();
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index 4f0081b6a7f..ea30fba9798 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance;
-import com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceException;
+import com.yahoo.config.provision.exception.LoadBalancerServiceException;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
 import com.yahoo.vespa.hosted.provision.lb.Real;
 import com.yahoo.vespa.hosted.provision.node.IP;