From 3b2ffb15084344fda6a444b271433c517c534028 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Tue, 3 Oct 2023 16:33:44 +0200 Subject: Normalize JSON --- .../hosted/provision/restapi/NodesV2ApiTest.java | 114 ++++++++++----------- .../hosted/provision/restapi/RestApiTester.java | 25 +++++ 2 files changed, 82 insertions(+), 57 deletions(-) (limited to 'node-repository') diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 04f83a8ec82..b862f3fa3b8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -54,26 +54,26 @@ public class NodesV2ApiTest { @Test public void test_requests() throws Exception { // GET - assertFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/state/"), "states.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/state/?recursive=true"), "states-recursive.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&includeDeprovisioned=true"), "nodes-recursive-include-deprovisioned.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?enclave=true"), "enclave-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&enclave=true"), "enclave-nodes-recursive.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/stats"), "stats.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/maintenance/"), "maintenance.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/wireguard/"), "wireguard.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/"), "root.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/state/"), "states.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/state/?recursive=true"), "states-recursive.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/state/active?recursive=true"), "active-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/"), "nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true"), "nodes-recursive.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&includeDeprovisioned=true"), "nodes-recursive-include-deprovisioned.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?enclave=true"), "enclave-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&enclave=true"), "enclave-nodes-recursive.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "node2.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/stats"), "stats.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/maintenance/"), "maintenance.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/wireguard/"), "wireguard.json"); // GET with filters - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&hostname=host6.yahoo.com%20host2.yahoo.com"), "application2-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterType=content"), "content-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&clusterId=id2"), "application2-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&application=tenant2.application2.instance2"), "application2-nodes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/?recursive=true&parentHost=dockerhost1.yahoo.com"), "child-nodes.json"); // POST restart command assertRestart(1, new Request("http://localhost:8080/nodes/v2/command/restart?hostname=host2.yahoo.com", @@ -105,10 +105,10 @@ public class NodesV2ApiTest { getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 4 nodes to the provisioned state\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // POST duplicate node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", @@ -235,7 +235,7 @@ public class NodesV2ApiTest { ((OrchestratorMock) tester.container().components().getComponent(OrchestratorMock.class.getName())) .suspend(new HostName("host4.yahoo.com")); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); // Park and remove host assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/dockerhost1.yahoo.com", @@ -265,11 +265,11 @@ public class NodesV2ApiTest { @Test public void test_application_requests() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1"), - "application1.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant2.application2.instance2"), - "application2.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1"), + "application1.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/application/tenant2.application2.instance2"), + "application2.json"); // Update (PATCH) an application assertResponse(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1", @@ -375,24 +375,24 @@ public class NodesV2ApiTest { @Test public void patch_hostnames() throws IOException { - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"additionalHostnames\": [\"a\",\"b\"]}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-with-hostnames.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-with-hostnames.json"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"additionalHostnames\": []}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); } @Test public void patch_wireguard_pubkey() throws IOException { - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4.json"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"wireguardPubkey\": \"not a wg key\"}"), Request.Method.PATCH), 400, @@ -402,7 +402,7 @@ public class NodesV2ApiTest { Utf8.toBytes("{\"wireguardPubkey\": \"lololololololololololololololololololololoo=\"}"), Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-wg.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-wg.json"); } @Test @@ -415,7 +415,7 @@ public class NodesV2ApiTest { Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/controller1.yahoo.com"), "controller1.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/controller1.yahoo.com"), "controller1.json"); } @Test @@ -460,12 +460,12 @@ public class NodesV2ApiTest { @Test public void acls_for_inclave_tenant_host() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/acl/host5.yahoo.com"), "acl-tenant-node.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/acl/host5.yahoo.com"), "acl-tenant-node.json"); } @Test public void acl_request_by_config_server() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/acl/cfg1.yahoo.com"), "acl-config-server.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/acl/cfg1.yahoo.com"), "acl-config-server.json"); } @Test @@ -588,7 +588,7 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Updated host5.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), "node5-after-changes.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com"), "node5-after-changes.json"); } @Test @@ -616,7 +616,7 @@ public class NodesV2ApiTest { "}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); // Patching with an empty reports is no-op tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", @@ -624,7 +624,7 @@ public class NodesV2ApiTest { Request.Method.PATCH), 200, "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports.json"); // Patching existing report overwrites tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", @@ -638,21 +638,21 @@ public class NodesV2ApiTest { Request.Method.PATCH), 200, "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-2.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-2.json"); // Clearing one report assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": { \"diskSpace\": null } }"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-3.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-3.json"); // Clearing all reports assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"reports\": null }"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); } @Test @@ -835,7 +835,7 @@ public class NodesV2ApiTest { Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); // Override wantedOsVersion assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", @@ -873,12 +873,12 @@ public class NodesV2ApiTest { "{\"message\":\"Will request firmware checks on all hosts.\"}"); // dockerhost1 displays both values. - assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "dockerhost1-with-firmware-data.json"); // host1 has no wantedFirmwareCheck, as it's not a docker host. - assertFile(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), - "node1.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "node1.json"); // Cancel the firmware check. assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/firmware", new byte[0], Request.Method.DELETE), @@ -887,8 +887,8 @@ public class NodesV2ApiTest { @Test public void test_capacity() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/capacity/?json=true"), "capacity-zone.json"); - assertFile(new Request("http://localhost:8080/nodes/v2/capacity?json=true"), "capacity-zone.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/capacity/?json=true"), "capacity-zone.json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/capacity?json=true"), "capacity-zone.json"); List hostsToRemove = List.of( "dockerhost1.yahoo.com", @@ -898,12 +898,12 @@ public class NodesV2ApiTest { ); String requestUriTemplate = "http://localhost:8080/nodes/v2/capacity/?json=true&hosts=%s"; - assertFile(new Request(String.format(requestUriTemplate, - String.join(",", hostsToRemove.subList(0, 3)))), - "capacity-hostremoval-possible.json"); - assertFile(new Request(String.format(requestUriTemplate, - String.join(",", hostsToRemove))), - "capacity-hostremoval-impossible.json"); + assertJsonFile(new Request(String.format(requestUriTemplate, + String.join(",", hostsToRemove.subList(0, 3)))), + "capacity-hostremoval-possible.json"); + assertJsonFile(new Request(String.format(requestUriTemplate, + String.join(",", hostsToRemove))), + "capacity-hostremoval-impossible.json"); } @@ -912,7 +912,7 @@ public class NodesV2ApiTest { public void test_single_node_rendering() throws Exception { for (int i = 1; i <= 14; i++) { if (i == 8 || i == 9 || i == 11 || i == 12) continue; // these nodes are added later - assertFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json"); + assertJsonFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json"); } } @@ -1116,8 +1116,8 @@ public class NodesV2ApiTest { tester.assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); } - private void assertFile(Request request, String file) throws IOException { - tester.assertFile(request, file); + private void assertJsonFile(Request request, String file) throws IOException { + tester.assertJsonFile(request, file); } private void assertResponse(Request request, String response) throws IOException { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java index 47745d25467..94f1e1e2509 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -8,6 +8,7 @@ import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.SystemName; import com.yahoo.io.IOUtils; +import com.yahoo.test.json.JsonTestHelper; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.ComparisonFailure; @@ -58,6 +59,30 @@ public class RestApiTester { responseSnippet, response), match, response.contains(responseSnippet)); } + public void assertJsonFile(Request request, String responseFile) throws IOException { + String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); + JsonTestHelper.Options options = new JsonTestHelper.Options().setCompact(true); + expectedResponse = include(expectedResponse); + expectedResponse = JsonTestHelper.normalize(options, expectedResponse); + expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace + String responseString = container.handleRequest(request).getBodyAsString(); + responseString = JsonTestHelper.normalize(options, responseString); + if (expectedResponse.contains("(ignore)")) { + // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches + // until the first stop character + String stopCharacters = "[^,:\\\\[\\\\]{}]"; + String expectedResponsePattern = Pattern.quote(expectedResponse) + .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + + stopCharacters + "*\\\\Q"); + if (!Pattern.matches(expectedResponsePattern, responseString)) { + throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, + responseString); + } + } else { + assertEquals(responseFile, expectedResponse, responseString); + } + } + public void assertFile(Request request, String responseFile) throws IOException { String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); expectedResponse = include(expectedResponse); -- cgit v1.2.3