From 891c8108223e878cfefafecd9df662c69ce14207 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 3 May 2019 13:27:17 +0200 Subject: Verify content hash if present in a header for submission --- .../restapi/application/ApplicationApiHandler.java | 19 ++++++++++++++++- .../restapi/application/MultipartParser.java | 22 +++++++++++++------- .../restapi/application/ApplicationApiTest.java | 24 ++++++++++++++++++++++ .../restapi/application/responses/jobs.json | 20 +++++++++--------- .../application/responses/system-test-job.json | 4 ++-- 5 files changed, 69 insertions(+), 20 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1b9bf28f395..c6568db0397 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; +import ai.vespa.hosted.api.Signatures; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; @@ -88,10 +89,13 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; +import java.security.DigestInputStream; import java.security.Principal; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Optional; @@ -1399,7 +1403,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse submit(String tenant, String application, HttpRequest request) { - Map dataParts = new MultipartParser().parse(request); + Map dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); SourceRevision sourceRevision = toSourceRevision(submitOptions); String authorEmail = submitOptions.field("authorEmail").asString(); @@ -1420,4 +1424,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP)); } + private static Map parseDataParts(HttpRequest request) { + String contentHash = request.getHeader("x-Content-Hash"); + if (contentHash == null) + return new MultipartParser().parse(request); + + DigestInputStream digester = Signatures.sha256Digester(request.getData()); + var dataParts = new MultipartParser().parse(request.getHeader("Content-Type"), digester, request.getUri()); + if ( ! Arrays.equals(digester.getMessageDigest().digest(), Base64.getDecoder().decode(contentHash))) + throw new IllegalArgumentException("Value of X-Content-Hash header does not match computed content hash"); + + return dataParts; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java index 75f4ff68f1e..3125a8a363a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/MultipartParser.java @@ -7,6 +7,8 @@ import org.apache.commons.fileupload.ParameterParser; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.net.URI; import java.util.HashMap; import java.util.Map; @@ -24,18 +26,24 @@ public class MultipartParser { * @throws IllegalArgumentException if this request is not a well-formed request with Content-Type multipart/form-data */ public Map parse(HttpRequest request) { + return parse(request.getHeader("Content-Type"), request.getData(), request.getUri()); + } + + /** + * Parses the given data stream for the given uri using the provided content-type header to determine boundaries. + * + * @throws IllegalArgumentException if this is not a well-formed request with Content-Type multipart/form-data + */ + public Map parse(String contentTypeHeader, InputStream data, URI uri) { try { ParameterParser parameterParser = new ParameterParser(); - Map contentType = parameterParser.parse(request.getHeader("Content-Type"), ';'); + Map contentType = parameterParser.parse(contentTypeHeader, ';'); if ( ! contentType.containsKey("multipart/form-data")) - throw new IllegalArgumentException("Expected a multipart message, but got Content-Type: " + - request.getHeader("Content-Type")); + throw new IllegalArgumentException("Expected a multipart message, but got Content-Type: " + contentTypeHeader); String boundary = contentType.get("boundary"); if (boundary == null) throw new IllegalArgumentException("Missing boundary property in Content-Type header"); - MultipartStream multipartStream = new MultipartStream(request.getData(), boundary.getBytes(), - 1000 * 1000, - null); + MultipartStream multipartStream = new MultipartStream(data, boundary.getBytes(), 1 << 20, null); boolean nextPart = multipartStream.skipPreamble(); Map parts = new HashMap<>(); while (nextPart) { @@ -55,7 +63,7 @@ public class MultipartParser { throw new IllegalArgumentException("Malformed multipart/form-data request", e); } catch(IOException e) { - throw new IllegalArgumentException("IO error reading multipart request " + request.getUri(), e); + throw new IllegalArgumentException("IO error reading multipart request " + uri, e); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7032112a860..891dc243406 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; +import ai.vespa.hosted.api.Signatures; import com.yahoo.application.container.handler.Request; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -74,6 +75,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -568,6 +570,21 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationSubmissionData(packageWithService)), "{\"message\":\"Application package version: 1.0.44-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + // Fourth attempt has a wrong content hash in a header, and fails. + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + .screwdriverIdentity(SCREWDRIVER_ID) + .header("X-Content-Hash", "not/the/right/hash") + .data(createApplicationSubmissionData(packageWithService)), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Value of X-Content-Hash header does not match computed content hash\"}", 400); + + // Fifth attempt has the right content hash in a header, and succeeds. + MultiPartStreamer streamer = createApplicationSubmissionData(packageWithService); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + .screwdriverIdentity(SCREWDRIVER_ID) + .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(streamer::data))) + .data(streamer), + "{\"message\":\"Application package version: 1.0.45-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); + ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) .screwdriverIdentity(SCREWDRIVER_ID) @@ -1618,6 +1635,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private AthenzIdentity identity; private OktaAccessToken oktaAccessToken; private String contentType = "application/json"; + private Map> headers = new HashMap<>(); private String recursive; private RequestBuilder(String path, Request.Method method) { @@ -1644,6 +1662,11 @@ public class ApplicationApiTest extends ControllerContainerTest { private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; } private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } + private RequestBuilder header(String name, String value) { + this.headers.putIfAbsent(name, new ArrayList<>()); + this.headers.get(name).add(value); + return this; + } @Override public Request get() { @@ -1651,6 +1674,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters (recursive == null ? "" : "?recursive=" + recursive), data, method); + request.getHeaders().addAll(headers); request.getHeaders().put("Content-Type", contentType); if (identity != null) { addIdentityToRequest(request, identity); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 5ac896e2753..72dd13474dc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -7,8 +7,8 @@ }, "application": { "application": { - "hash": "1.0.44-d00d", - "build": 44, + "hash": "1.0.45-d00d", + "build": 45, "source": { "gitRepository": "repo", "gitBranch": "master", @@ -21,8 +21,8 @@ }, "deploying": { "application": { - "hash": "1.0.44-d00d", - "build": 44, + "hash": "1.0.45-d00d", + "build": 45, "source": { "gitRepository": "repo", "gitBranch": "master", @@ -42,8 +42,8 @@ "start": (ignore), "wantedPlatform": "6.1", "wantedApplication": { - "hash": "1.0.44-d00d", - "build": 44, + "hash": "1.0.45-d00d", + "build": 45, "source": { "gitRepository": "repo", "gitBranch": "master", @@ -78,8 +78,8 @@ "start": (ignore), "wantedPlatform": "6.1", "wantedApplication": { - "hash": "1.0.44-d00d", - "build": 44, + "hash": "1.0.45-d00d", + "build": 45, "source": { "gitRepository": "repo", "gitBranch": "master", @@ -112,8 +112,8 @@ "status": "pending", "wantedPlatform": "6.1", "wantedApplication": { - "hash": "1.0.44-d00d", - "build": 44, + "hash": "1.0.45-d00d", + "build": 45, "source": { "gitRepository": "repo", "gitBranch": "master", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index 2f5c6e489bd..107d969e8ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -5,8 +5,8 @@ "start": (ignore), "wantedPlatform": "6.1", "wantedApplication": { - "hash": "1.0.44-d00d", - "build": 44, + "hash": "1.0.45-d00d", + "build": 45, "source": { "gitRepository": "repo", "gitBranch": "master", -- cgit v1.2.3 From c4a609eee2f5dca31435c7395af466e414ecaa89 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 3 May 2019 13:35:16 +0200 Subject: Let buildService inherit applicationReader --- .../yahoo/vespa/hosted/controller/api/role/RoleDefinition.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index af068decc83..980b8bd316f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -20,10 +20,6 @@ public enum RoleDefinition { /** Deus ex machina. */ hostedOperator(Policy.operator), - /** Build service which may submit new applications for continuous deployment. */ - buildService(Policy.submission, - Policy.applicationRead), - /** Base role which every user is part of. */ everyone(Policy.classifiedRead, Policy.publicRead, @@ -36,6 +32,10 @@ public enum RoleDefinition { Policy.applicationRead, Policy.deploymentRead), + /** Build service which may submit new applications for continuous deployment. */ + buildService(applicationReader, + Policy.submission), + /** Application developer with access to deploy to development zones. */ applicationDeveloper(applicationReader, Policy.developmentDeployment), -- cgit v1.2.3 From cccb67efbb512b4de13ba44243d6d8f728c00e84 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 3 May 2019 13:37:16 +0200 Subject: Let key authentication imply applicationDeveloper role as well --- .../vespa/hosted/controller/restapi/filter/SignatureFilter.java | 3 ++- .../hosted/controller/restapi/filter/SignatureFilterTest.java | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 5cf29179d2a..0526c69e2bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -59,7 +59,8 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase { if (verified) request.setAttribute(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(() -> "buildService@" + id.tenant() + "." + id.application(), - Set.of(Role.buildService(id.tenant(), id.application())))); + Set.of(Role.buildService(id.tenant(), id.application()), + Role.applicationDeveloper(id.tenant(), id.application())))); } catch (Exception e) { logger.log(LogLevel.DEBUG, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index bf44481c110..970cd6071d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -83,7 +83,9 @@ public class SignatureFilterTest { assertTrue(filter.filter(signed).isEmpty()); SecurityContext securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME); assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName()); - assertEquals(Set.of(Role.buildService(id.tenant(), id.application())), securityContext.roles()); + assertEquals(Set.of(Role.buildService(id.tenant(), id.application()), + Role.applicationDeveloper(id.tenant(), id.application())), + securityContext.roles()); // Signed POST request also gets a build service role. byte[] hiBytes = new byte[]{0x48, 0x69}; @@ -91,7 +93,9 @@ public class SignatureFilterTest { filter.filter(signed); securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME); assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName()); - assertEquals(Set.of(Role.buildService(id.tenant(), id.application())), securityContext.roles()); + assertEquals(Set.of(Role.buildService(id.tenant(), id.application()), + Role.applicationDeveloper(id.tenant(), id.application())), + securityContext.roles()); // Unsigned requests still get no roles. filter.filter(unsigned); -- cgit v1.2.3 From 55917c189bcb480644a8e2c5a8c861234502d85a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 3 May 2019 13:54:25 +0200 Subject: Copyright headers --- .../hosted/controller/restapi/application/ApplicationApiHandler.java | 2 +- .../yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java | 1 + .../vespa/hosted/controller/restapi/filter/SignatureFilterTest.java | 1 + hosted-api/src/main/java/ai/vespa/hosted/api/Method.java | 1 + hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java | 1 + hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java | 1 + hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java | 1 + hosted-api/src/main/java/ai/vespa/hosted/api/Signatures.java | 1 + hosted-api/src/test/java/ai/vespa/hosted/api/MultiPartStreamerTest.java | 1 + hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java | 1 + 10 files changed, 10 insertions(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index c6568db0397..5c5ed88d916 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -915,7 +915,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ZoneId zone = ZoneId.from(environment, region); // Get deployOptions - Map dataParts = new MultipartParser().parse(request); + Map dataParts = parseDataParts(request); if ( ! dataParts.containsKey("deployOptions")) return ErrorResponse.badRequest("Missing required form part 'deployOptions'"); Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 0526c69e2bd..214413441d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import ai.vespa.hosted.api.Method; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 970cd6071d0..0bddae11572 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.filter; import ai.vespa.hosted.api.Method; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Method.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Method.java index ff7c1e4270b..8c50fc665b3 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Method.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Method.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; /** diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java index 0dde6fd3bde..c7aeca92ad0 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import java.io.BufferedInputStream; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java index 48ff10695d3..b128c837caa 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestSigner.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.security.KeyUtils; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java index 1d672a56dcb..a578b35c35a 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.security.KeyUtils; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Signatures.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Signatures.java index c93d5fc9168..74a4bb6099f 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Signatures.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Signatures.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import com.yahoo.security.KeyUtils; diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/MultiPartStreamerTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/MultiPartStreamerTest.java index d94a5b3314c..a55c0d91cd3 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/MultiPartStreamerTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/MultiPartStreamerTest.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import org.junit.Rule; diff --git a/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java b/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java index 0ac46a51a43..9be32812514 100644 --- a/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java +++ b/hosted-api/src/test/java/ai/vespa/hosted/api/SignaturesTest.java @@ -1,3 +1,4 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.api; import org.junit.Test; -- cgit v1.2.3 From 9be7c1d44d74f0d3a59bb02009799c11088d2be4 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 3 May 2019 13:55:41 +0200 Subject: No more need to httpmime --- controller-server/pom.xml | 6 --- .../restapi/application/ApplicationApiTest.java | 48 ++++++++-------------- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 3e7247bd44b..c6c6acafe15 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -159,12 +159,6 @@ test - - org.apache.httpcomponents - httpmime - test - - com.github.tomakehurst wiremock-standalone diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 891dc243406..76572d1fd0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -61,13 +61,9 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.yolean.Exceptions; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.mime.MultipartEntityBuilder; import org.junit.Before; import org.junit.Test; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -210,7 +206,7 @@ public class ApplicationApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); // POST (deploy) an application to a zone - manual user deployment - HttpEntity entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -689,7 +685,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy ApplicationId id = createTenantAndApplication(); long projectId = 1; - HttpEntity deployData = createApplicationDeployData(Optional.empty(), false); + MultiPartStreamer deployData = createApplicationDeployData(Optional.empty(), false); startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-west-1 @@ -771,14 +767,14 @@ public class ApplicationApiTest extends ControllerContainerTest { new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId("application1")); // POST (deploy) an application to a prod zone - allowed when project ID is not specified - HttpEntity entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/deploy", POST) .data(entity) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); // POST (deploy) a system application with an application package - HttpEntity noAppEntity = createApplicationDeployData(Optional.empty(), true); + MultiPartStreamer noAppEntity = createApplicationDeployData(Optional.empty(), true); tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/routing/environment/prod/region/us-central-1/instance/default/deploy", POST) .data(noAppEntity) .userIdentity(HOSTED_VESPA_OPERATOR), @@ -807,7 +803,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); ApplicationId id = createTenantAndApplication(); long projectId = 1; - HttpEntity deployData = createApplicationDeployData(Optional.empty(), false); + MultiPartStreamer deployData = createApplicationDeployData(Optional.empty(), false); startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-east-3 @@ -952,7 +948,7 @@ public class ApplicationApiTest extends ControllerContainerTest { configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE, null)); // POST (deploy) an application with an invalid application package - HttpEntity entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1078,7 +1074,7 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Deploy to an authorized zone by a user tenant is disallowed - HttpEntity entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) .data(entity) .userIdentity(USER_ID), @@ -1204,7 +1200,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application to a dev zone String expectedResult="{\"error-code\":\"BAD_REQUEST\",\"message\":\"User user.new-user is not allowed to launch services in Athenz domain domain1. Please reach out to the domain admin.\"}"; - HttpEntity entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) .data(entity) .userIdentity(userId), @@ -1237,7 +1233,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); // POST (deploy) an application to a dev zone - HttpEntity entity = createApplicationDeployData(applicationPackage, true); + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) .data(entity) .userIdentity(tenantAdmin), @@ -1420,20 +1416,20 @@ public class ApplicationApiTest extends ControllerContainerTest { } } - private HttpEntity createApplicationDeployData(ApplicationPackage applicationPackage, boolean deployDirectly) { + private MultiPartStreamer createApplicationDeployData(ApplicationPackage applicationPackage, boolean deployDirectly) { return createApplicationDeployData(Optional.of(applicationPackage), deployDirectly); } - private HttpEntity createApplicationDeployData(Optional applicationPackage, boolean deployDirectly) { + private MultiPartStreamer createApplicationDeployData(Optional applicationPackage, boolean deployDirectly) { return createApplicationDeployData(applicationPackage, Optional.empty(), deployDirectly); } - private HttpEntity createApplicationDeployData(Optional applicationPackage, + private MultiPartStreamer createApplicationDeployData(Optional applicationPackage, Optional applicationVersion, boolean deployDirectly) { - MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.addTextBody("deployOptions", deployOptions(deployDirectly, applicationVersion), ContentType.APPLICATION_JSON); - applicationPackage.ifPresent(ap -> builder.addBinaryBody("applicationZip", ap.zippedContent())); - return builder.build(); + MultiPartStreamer streamer = new MultiPartStreamer(); + streamer.addJson("deployOptions", deployOptions(deployDirectly, applicationVersion)); + applicationPackage.ifPresent(ap -> streamer.addBytes("applicationZip", ap.zippedContent())); + return streamer; } private MultiPartStreamer createApplicationSubmissionData(ApplicationPackage applicationPackage) { @@ -1509,7 +1505,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private void startAndTestChange(ContainerControllerTester controllerTester, ApplicationId application, long projectId, ApplicationPackage applicationPackage, - HttpEntity deployData, long buildNumber) { + MultiPartStreamer deployData, long buildNumber) { ContainerTester tester = controllerTester.containerTester(); // Trigger application change @@ -1648,15 +1644,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private RequestBuilder data(MultiPartStreamer streamer) { return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType())); } - private RequestBuilder data(HttpEntity data) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - data.writeTo(out); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return data(out.toByteArray()).contentType(data.getContentType().getValue()); - } + private RequestBuilder userIdentity(UserId userId) { this.identity = HostedAthenzIdentities.from(userId); return this; } private RequestBuilder screwdriverIdentity(ScrewdriverId screwdriverId) { this.identity = HostedAthenzIdentities.from(screwdriverId); return this; } private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; } -- cgit v1.2.3 From 0999cc21b472b3d302b47ab331c4b243ab098c84 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 3 May 2019 13:59:25 +0200 Subject: Verify content hash for deployments too --- .../hosted/controller/restapi/application/ApplicationApiTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 76572d1fd0c..3fcead5d0b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -205,10 +205,11 @@ public class ApplicationApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); - // POST (deploy) an application to a zone - manual user deployment + // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification) MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) .data(entity) + .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data))) .userIdentity(USER_ID), new File("deploy-result.json")); -- cgit v1.2.3