Proxy state/v1/ requests with host

8 files changed, 117 insertions, 31 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index c4731ef0860..32422889b48 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -560,8 +560,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         }
     }
 
-    public HttpResponse proxyServiceHostnameRequest(ApplicationId applicationId, String hostName, String serviceName, HttpURL.Path path, Query query) {
-        return httpProxy.get(getApplication(applicationId), hostName, serviceName, path, query);
+    public HttpResponse proxyServiceHostnameRequest(ApplicationId applicationId, String hostName, String serviceName, HttpURL.Path path, Query query, HttpURL forwardedUrl) {
+        return httpProxy.get(getApplication(applicationId), hostName, serviceName, path, query, forwardedUrl);
     }
 
     public Map<String, ClusterReindexing> getClusterReindexingStatus(ApplicationId applicationId) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java
index cc022c93278..8e758967fb5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java
@@ -11,16 +11,22 @@ import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.vespa.config.server.http.HttpErrorResponse;
 import com.yahoo.vespa.config.server.http.HttpFetcher;
 import com.yahoo.vespa.config.server.http.HttpFetcher.Params;
 import com.yahoo.vespa.config.server.http.NotFoundException;
 import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 
-import java.net.MalformedURLException;
+import java.io.ByteArrayOutputStream;
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
-import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 public class HttpProxy {
 
@@ -35,6 +41,10 @@ public class HttpProxy {
     }
 
     public HttpResponse get(Application application, String hostName, String serviceType, Path path, Query query) {
+        return get(application, hostName, serviceType, path, query, null);
+    }
+
+    public HttpResponse get(Application application, String hostName, String serviceType, Path path, Query query, HttpURL forwardedUrl) {
         HostInfo host = application.getModel().getHosts().stream()
                 .filter(hostInfo -> hostInfo.getHostname().equals(hostName))
                 .findFirst()
@@ -52,18 +62,39 @@ public class HttpProxy {
                                .findFirst()
                                .orElseThrow(() -> new NotFoundException("Failed to find HTTP state port"));
 
-        return internalGet(host.getHostname(), port.getPort(), path, query);
+        HttpURL url = HttpURL.create(Scheme.http, DomainName.of(host.getHostname()), port.getPort(), path, query);
+        HttpResponse response = fetcher.get(new Params(2000), // 2_000 ms read timeout
+                                            url.asURI());
+        return forwardedUrl == null ? response : new UrlRewritingProxyResponse(response, url, forwardedUrl);
     }
 
-    private HttpResponse internalGet(String hostname, int port, Path path, Query query) {
-        HttpURL url = HttpURL.create(Scheme.http, DomainName.of(hostname), port, path, query);
-        try {
-            return fetcher.get(new Params(2000), // 2_000 ms read timeout
-                               url.asURI().toURL());
-        } catch (MalformedURLException e) {
-            logger.log(Level.WARNING, "Badly formed url: " + url, e);
-            return HttpErrorResponse.internalServerError("Failed to construct URL for backend");
+    static class UrlRewritingProxyResponse extends HttpResponse {
+
+        final HttpResponse wrapped;
+        final String patten;
+        final String replacement;
+
+        public UrlRewritingProxyResponse(HttpResponse wrapped, HttpURL requestUrl, HttpURL forwardedUrl) {
+            super(wrapped.getStatus());
+            this.wrapped = wrapped;
+            this.patten = Pattern.quote(requestUrl.withPath(requestUrl.path().withoutTrailingSlash()).withQuery(Query.empty()).asURI().toString());
+            this.replacement = forwardedUrl.withPath(forwardedUrl.path().withoutTrailingSlash()).withQuery(Query.empty()).asURI().toString();
         }
+
+        @Override
+        public void render(OutputStream outputStream) throws IOException {
+            ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+            wrapped.render(buffer);
+            outputStream.write(buffer.toString(Charset.forName(wrapped.getCharacterEncoding()))
+                                     .replaceAll(patten, replacement)
+                                     .getBytes(UTF_8));
+        }
+
+        @Override
+        public String getContentType() {
+            return wrapped.getContentType();
+        }
+
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java
index 6c05e755721..df5b7b74f7a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpFetcher.java
@@ -3,9 +3,11 @@ package com.yahoo.vespa.config.server.http;
 
 import com.yahoo.container.jdisc.HttpResponse;
 
+import java.net.URI;
 import java.net.URL;
 
 public interface HttpFetcher {
+
     class Params {
         // See HttpUrlConnection::setReadTimeout. 0 means infinite (not recommended!).
         public final int readTimeoutMs;
@@ -16,5 +18,6 @@ public interface HttpFetcher {
     }
 
     // On failure to get or build HttpResponse for url, an exception is thrown to be handled by HttpHandler.
-    HttpResponse get(Params params, URL url);
+    HttpResponse get(Params params, URI url);
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java
index 6add1f7a9fc..724b9417dc1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java
@@ -12,6 +12,7 @@ import org.apache.hc.core5.util.Timeout;
 
 import java.io.IOException;
 import java.net.SocketTimeoutException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.logging.Level;
@@ -23,9 +24,9 @@ public class SimpleHttpFetcher implements HttpFetcher {
     private final CloseableHttpClient client = VespaHttpClientBuilder.create().build();
 
     @Override
-    public HttpResponse get(Params params, URL url) {
+    public HttpResponse get(Params params, URI url) {
         try {
-            HttpGet request = new HttpGet(url.toURI());
+            HttpGet request = new HttpGet(url);
             request.addHeader("Connection", "Close");
             request.setConfig(
                     RequestConfig.custom()
@@ -47,10 +48,7 @@ public class SimpleHttpFetcher implements HttpFetcher {
             String message = "Failed to get response from " + url;
             logger.log(Level.WARNING, message, e);
             throw new InternalServerException(message);
-        } catch (URISyntaxException e) {
-            String message = "Invalid URL: " + e.getMessage();
-            logger.log(Level.WARNING, message, e);
-            throw new InternalServerException(message, e);
         }
     }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java
index a64a5551095..b4c3647d9e1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/StaticResponse.java
@@ -8,6 +8,7 @@ import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 
 public class StaticResponse extends HttpResponse {
+
     private final String contentType;
     private final byte[] body;
 
@@ -30,4 +31,5 @@ public class StaticResponse extends HttpResponse {
     public String getContentType() {
         return contentType;
     }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 813933a5d9b..9af5db48dd0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -39,6 +39,7 @@ import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
@@ -87,7 +88,7 @@ public class ApplicationHandler extends HttpHandler {
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return protonMetrics(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request);
-        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/state/v1/metrics")) return serviceStateV1metrics(applicationId(path), path.get("service"), path.get("hostname"));
+        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/state/v1/{*}")) return serviceStateV1(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge")) return listServiceConverge(applicationId(path), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path));
@@ -147,11 +148,16 @@ public class ApplicationHandler extends HttpHandler {
             default:
                 throw new com.yahoo.vespa.config.server.NotFoundException("No status page for service: " + service);
         }
-        return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, pathPrefix.append(pathSuffix), Query.empty().add(request.getJDiscRequest().parameters()));
+        return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, pathPrefix.append(pathSuffix), Query.empty().add(request.getJDiscRequest().parameters()), null);
     }
 
-    private HttpResponse serviceStateV1metrics(ApplicationId applicationId, String service, String hostname) {
-        return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, HttpURL.Path.parse("/state/v1/metrics"), Query.empty());
+    private HttpResponse serviceStateV1(ApplicationId applicationId, String service, String hostname, HttpURL.Path rest, HttpRequest request) {
+        Query query = Query.empty().add(request.getJDiscRequest().parameters());
+        String forwardedUrl = query.lastEntries().get("forwarded-url");
+        return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service,
+                                                                 HttpURL.Path.parse("/state/v1").append(rest),
+                                                                 query.remove("forwarded-url"),
+                                                                 forwardedUrl == null ? null : HttpURL.from(URI.create(forwardedUrl)));
     }
 
     private HttpResponse content(ApplicationId applicationId, HttpURL.Path contentPath, HttpRequest request) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java
index 3e934e5e19e..80e998521a9 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.application;
 
+import ai.vespa.http.HttpURL;
 import ai.vespa.http.HttpURL.Query;
 import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.Model;
@@ -8,6 +9,8 @@ import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.container.jdisc.HttpResponse;
 import ai.vespa.http.HttpURL.Path;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.http.HttpFetcher;
 import com.yahoo.vespa.config.server.http.RequestTimeoutException;
 import com.yahoo.vespa.config.server.http.StaticResponse;
@@ -15,15 +18,22 @@ import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
 import static com.yahoo.vespa.config.server.application.MockModel.createServiceInfo;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertSame;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -45,7 +55,7 @@ public class HttpProxyTest {
     @Test
     public void testNormalGet() throws Exception {
         ArgumentCaptor<HttpFetcher.Params> actualParams = ArgumentCaptor.forClass(HttpFetcher.Params.class);
-        ArgumentCaptor<URL> actualUrl = ArgumentCaptor.forClass(URL.class);
+        ArgumentCaptor<URI> actualUrl = ArgumentCaptor.forClass(URI.class);
         HttpResponse response = new StaticResponse(200, "application/json", "body");
         when(fetcher.get(actualParams.capture(), actualUrl.capture())).thenReturn(response);
 
@@ -57,14 +67,42 @@ public class HttpProxyTest {
         assertEquals(2000, actualParams.getValue().readTimeoutMs);
 
         assertEquals(1, actualUrl.getAllValues().size());
-        assertEquals(new URL("http://" + hostname + ":" + port + "/clustercontroller-status/v1/clusterName?foo=bar"),
-                actualUrl.getValue());
+        assertEquals(URI.create("http://" + hostname + ":" + port + "/clustercontroller-status/v1/clusterName?foo=bar"),
+                     actualUrl.getValue());
 
         // The HttpResponse returned by the fetcher IS the same object as the one returned by the proxy,
         // when everything goes well.
         assertSame(actualResponse, response);
     }
 
+    static String toJson(URI uri, String morePath) throws IOException {
+        Slime slime = new Slime();
+        slime.setObject().setString("url", HttpURL.from(uri).appendPath(Path.parse(morePath)).asURI().toString());
+        return new String(SlimeUtils.toJsonBytes(slime), UTF_8);
+    }
+
+    @Test
+    public void testNormalGetWithRewrite() throws Exception {
+        ArgumentCaptor<HttpFetcher.Params> actualParams = ArgumentCaptor.forClass(HttpFetcher.Params.class);
+        ArgumentCaptor<URI> actualUrl = ArgumentCaptor.forClass(URI.class);
+        doAnswer(invoc -> new StaticResponse(200, "application/json",
+                                             toJson(invoc.getArgument(1, URI.class), "/nested/path")))
+                .when(fetcher).get(actualParams.capture(), actualUrl.capture());
+
+        HttpResponse actualResponse = proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName,
+                                                Path.parse("/service/path"),
+                                                Query.parse("foo=%2F"),
+                                                HttpURL.from(URI.create("https://api:666/api/path%2E/with?foo=%2F")));
+
+        assertEquals(1, actualUrl.getAllValues().size());
+        assertEquals(URI.create("http://" + hostname + ":" + port + "/service/path?foo=%2F"),
+                     actualUrl.getValue());
+
+        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+        actualResponse.render(buffer);
+        assertEquals("{\"url\":\"https://api:666/api/path./with/nested/path?foo=%2F\"}", buffer.toString(UTF_8));
+    }
+
     @Test(expected = RequestTimeoutException.class)
     public void testFetchException() {
         when(fetcher.get(any(), any())).thenThrow(new RequestTimeoutException("timed out"));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 856e778942b..d132f0dca03 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -55,6 +55,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.net.URI;
+import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.time.Instant;
@@ -78,6 +79,7 @@ import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCod
 import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
 import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceListResponse;
 import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceResponse.createResponse;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -364,12 +366,18 @@ public class ApplicationHandlerTest {
                                              "service=" + invoc.getArgument(2, String.class) + "," +
                                              "path=" + invoc.getArgument(3, HttpURL.Path.class) + "," +
                                              "query=" + invoc.getArgument(4, HttpURL.Query.class) +
+                                             (invoc.getArgument(5, HttpURL.class) == null ? "" : ("," +
+                                             "forwardedUrl=" + invoc.getArgument(5, HttpURL.class))) +
                                              "</html>"))
-                .when(mockHttpProxy).get(any(), any(), any(), any(), any());
+                .when(mockHttpProxy).get(any(), any(), any(), any(), any(), any());
 
         HttpResponse response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/container-clustercontroller/" + host + "/status/some/path/clusterName1?foo=bar", GET));
         assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=container-clustercontroller,path=path '/clustercontroller-status/v1/some/path/clusterName1',query=query 'foo=bar'</html>");
 
+        String forwarded = "https://api:123/my/base/path?bar=%2E";
+        response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/distributor/" + host + "/state/v1/something/more?foo=bar&forwarded-url=" + URLEncoder.encode(forwarded, UTF_8), GET));
+        assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=path '/state/v1/something/more',query=query 'foo=bar',forwardedUrl=https://api:123/my/base/path?bar=.</html>");
+
         response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/distributor/" + host + "/status/something?foo=bar", GET));
         assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=path '/something',query=query 'foo=bar'</html>");
 
@@ -419,7 +427,7 @@ public class ApplicationHandlerTest {
         var mockHandler = createApplicationHandler();
 
         var requestString = "{\"name\":\"store\",\"awsId\":\"aws-id\",\"role\":\"role\",\"region\":\"us-west-1\",\"parameterName\":\"some-parameter\"}";
-        var requestData = new ByteArrayInputStream(requestString.getBytes(StandardCharsets.UTF_8));
+        var requestData = new ByteArrayInputStream(requestString.getBytes(UTF_8));
         var response = mockHandler.handle(createTestRequest(url, POST, requestData));
         assertEquals(200, response.getStatus());
 
@@ -455,7 +463,7 @@ public class ApplicationHandlerTest {
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/run/staging-test";
         ApplicationHandler mockHandler = createApplicationHandler();
 
-        InputStream requestData =  new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8));
+        InputStream requestData =  new ByteArrayInputStream("foo".getBytes(UTF_8));
         HttpRequest testRequest = createTestRequest(url, POST, requestData);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());