diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-12-02 12:08:22 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-12-02 12:18:53 +0000 |
commit | ed16494373ec67881f4389cb72379eb84cb39bd2 (patch) | |
tree | 93a5472bf8c74ea0c8eb8bba20aa52b41772be97 /vespaclient-java | |
parent | e88b274e49c2b74495caa210621ac0cc47d797cc (diff) |
Move vespa-status-filedistribution to where the other clients are and use a common jar file.
Diffstat (limited to 'vespaclient-java')
5 files changed, 412 insertions, 0 deletions
diff --git a/vespaclient-java/CMakeLists.txt b/vespaclient-java/CMakeLists.txt index 2af1bbe55cd..666e9633726 100644 --- a/vespaclient-java/CMakeLists.txt +++ b/vespaclient-java/CMakeLists.txt @@ -12,3 +12,4 @@ vespa_install_script(src/main/sh/vespa-get.sh vespa-get bin) vespa_install_script(src/main/sh/vespa-visit.sh vespa-visit bin) vespa_install_script(src/main/sh/vespa-visit-target.sh vespa-visit-target bin) vespa_install_script(src/main/sh/vespa-feed-perf vespa-feed-perf bin) +vespa_install_script(src/main/sh/vespa-status-filedistribution.sh vespa-status-filedistribution bin) diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index acfc7834ef2..f5856e1d019 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -54,9 +54,21 @@ <artifactId>lz4-java</artifactId> </dependency> <dependency> + <groupId>io.airlift</groupId> + <artifactId>airline</artifactId> + </dependency> + <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> + <dependency> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents.core5</groupId> + <artifactId>httpcore5</artifactId> + </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter</artifactId> diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java b/vespaclient-java/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java new file mode 100644 index 00000000000..a9bbfafeaf0 --- /dev/null +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java @@ -0,0 +1,249 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution.status; + +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.airlift.airline.Command; +import io.airlift.airline.HelpOption; +import io.airlift.airline.Option; +import io.airlift.airline.SingleCommand; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.util.Timeout; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; + +import static org.apache.hc.client5.http.config.RequestConfig.custom; + +/** + * Tool for getting file distribution status + * + * @author hmusum + */ +public class FileDistributionStatusClient { + + private static final String statusUnknown = "UNKNOWN"; + private static final String statusInProgress = "IN_PROGRESS"; + private static final String statusFinished = "FINISHED"; + + private final String tenantName; + private final String applicationName; + private final String instanceName; + private final String environment; + private final String region; + private final double timeout; + private final boolean debug; + + FileDistributionStatusClient(CommandLineArguments arguments) { + tenantName = arguments.getTenantName(); + applicationName = arguments.getApplicationName(); + instanceName = arguments.getInstanceName(); + environment = arguments.getEnvironment(); + region = arguments.getRegion(); + timeout = arguments.getTimeout(); + debug = arguments.getDebugFlag(); + } + + public static void main(String[] args) { + try { + new FileDistributionStatusClient(CommandLineArguments.build(args)).run(); + } catch (Exception e) { + System.err.println(e.getMessage()); + System.exit(1); + } + } + + public void run() { + String json = doHttpRequest(); + System.out.println(parseAndGenerateOutput(json)); + } + + private String doHttpRequest() { + Timeout timeoutInMillis = Timeout.ofMilliseconds((long) (timeout * 1000)); + RequestConfig config = custom() + .setConnectTimeout(timeoutInMillis) + .setConnectionRequestTimeout(timeoutInMillis) + .setResponseTimeout(timeoutInMillis) + .build(); + CloseableHttpClient httpClient = VespaHttpClientBuilder.create().build(); + URI statusUri = createStatusApiUri(); + if (debug) + System.out.println("URI:" + statusUri); + try { + HttpGet request = new HttpGet(statusUri); + request.addHeader("Connection", "Close"); + request.setConfig(config); + CloseableHttpResponse response = httpClient.execute(request); + HttpEntity entity = response.getEntity(); + String content = EntityUtils.toString(entity); + if (debug) + System.out.println("response:" + content); + if (response.getCode() == 200) { + return content; + } else { + throw new RuntimeException("Failed to get status for request " + statusUri + ": " + + response.getCode() + ": " + content); + } + } catch (IOException | ParseException e) { + throw new RuntimeException(e); + } + } + + String parseAndGenerateOutput(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (IOException e) { + throw new RuntimeException(e); + } + String status = jsonNode.get("status").asText(); + switch (status) { + case statusUnknown: + return "File distribution status unknown: " + jsonNode.get("message").asText(); + case statusInProgress: + return "File distribution in progress:\n" + inProgressOutput(jsonNode.get("hosts")); + case statusFinished: + return "File distribution finished"; + default: + throw new RuntimeException("Unknown status " + status); + } + } + + private URI createStatusApiUri() { + String path = String.format("/application/v2/tenant/%s/application/%s/environment/%s/region/%s/instance/%s/filedistributionstatus", + tenantName, applicationName, environment, region, instanceName); + try { + return new URIBuilder() + .setScheme("http") + .setHost("localhost") + .setPort(19071) + .setPath(path) + .addParameter("timeout", String.valueOf(timeout)) + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private String inProgressOutput(JsonNode hosts) { + ArrayList<String> statusPerHost = new ArrayList<>(); + for (JsonNode host : hosts) { + String status = host.get("status").asText(); + StringBuilder sb = new StringBuilder(host.get("hostname").asText()).append(": ").append(status); + switch (status) { + case statusUnknown: + sb.append(" (").append(host.get("message").asText()).append(")"); + break; + case statusInProgress: + JsonNode fileReferencesArray = host.get("fileReferences"); + int finished = 0; + for (JsonNode element : fileReferencesArray) { + for (Iterator<Map.Entry<String, JsonNode>> it = element.fields(); it.hasNext(); ) { + Map.Entry<String, JsonNode> fileReferenceStatus = it.next(); + if (fileReferenceStatus.getValue().asDouble() == 1.0) + finished++; + } + } + sb.append(" (" + finished + " of " + fileReferencesArray.size() + " finished)"); + break; + case statusFinished: + break; // Nothing to add + default: + throw new RuntimeException("Unknown status " + status); + } + statusPerHost.add(sb.toString()); + } + return String.join("\n", statusPerHost); + } + + @Command(name = "vespa-status-filedistribution", description = "Tool for getting file distribution status.") + public static class CommandLineArguments { + + static CommandLineArguments build(String[] args) { + CommandLineArguments arguments = null; + try { + arguments = SingleCommand.singleCommand(CommandLineArguments.class).parse(args); + } catch (Exception e) { + System.err.println(e.getMessage()); + System.err.println("Use --help to show usage.\n"); + System.exit(1); + } + + if (arguments.helpOption.showHelpIfRequested()) { + System.exit(0); + } + + if (arguments.getTenantName() == null) { + System.err.println("'--tenant' not set."); + System.exit(1); + } + + if (arguments.getApplicationName() == null) { + System.err.println("'--application' not set."); + System.exit(1); + } + + return arguments; + } + + @Inject + HelpOption helpOption; + + @Option(name = {"--tenant"}, + description = "tenant name") + private String tenantNameArg; + + @Option(name = {"--application"}, + description = "application name") + private String applicationNameArg; + + @Option(name = {"--instance"}, + description = "instance name") + private String instanceNameArg = "default"; + + @Option(name = {"--environment"}, + description = "environment name") + private String environmentArg = "prod"; + + @Option(name = {"--region"}, + description = "region name") + private String regionArg = "default"; + + @Option(name = {"--timeout"}, + description = "The timeout (in seconds).") + private double timeoutArg = 5; + + @Option(name = {"--debug"}, + description = "Print debug log.") + private boolean debugArg; + + public String getTenantName() { return tenantNameArg; } + + public String getApplicationName() { return applicationNameArg; } + + public String getInstanceName() { return instanceNameArg; } + + public String getEnvironment() { return environmentArg; } + + public String getRegion() { return regionArg; } + + public double getTimeout() { return timeoutArg; } + + public boolean getDebugFlag() { return debugArg; } + } + +} diff --git a/vespaclient-java/src/main/sh/vespa-status-filedistribution.sh b/vespaclient-java/src/main/sh/vespa-status-filedistribution.sh new file mode 100644 index 00000000000..70976581723 --- /dev/null +++ b/vespaclient-java/src/main/sh/vespa-status-filedistribution.sh @@ -0,0 +1,91 @@ +#!/bin/sh +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + empty_if_start_slash=${mypath%%/*} + if [ "${empty_if_start_slash}" ]; then + mypath=$(pwd)/${mypath} + fi + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findhost () { + if [ "${VESPA_HOSTNAME}" = "" ]; then + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 + fi + export VESPA_HOSTNAME +} + +findroot +findhost + +ROOT=${VESPA_HOME%/} +export ROOT + +# END environment bootstrap section + +if [ "$VESPA_ENVIRONMENT" != "" ]; then + environment="--environment $VESPA_ENVIRONMENT" +fi +if [ "$VESPA_REGION" != "" ]; then + region="--region $VESPA_REGION" +fi + +defaults="--tenant default --application default --instance default" +jvmoptions="-XX:MaxJavaStackTraceDepth=1000000 $(getJavaOptionsIPV46) -Xms48m -Xmx48m" +jar="-cp ${VESPA_HOME}/lib/jars/vespaclient-java-jar-with-dependencies.jar" + +exec java $jvmoptions $jar com.yahoo.vespa.filedistribution.status.FileDistributionStatusClient $defaults $environment $region "$@" diff --git a/vespaclient-java/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java b/vespaclient-java/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java new file mode 100644 index 00000000000..a724838e7c7 --- /dev/null +++ b/vespaclient-java/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution.status; + +import org.junit.Test; + +import static com.yahoo.vespa.filedistribution.status.FileDistributionStatusClient.CommandLineArguments; +import static org.junit.Assert.assertEquals; + +public class FileDistributionStatusClientTest { + + private static final CommandLineArguments arguments = createArguments("--tenant", "foo", "--application", "bar"); + private final FileDistributionStatusClient client = new FileDistributionStatusClient(arguments); + + @Test + public void finishedForAllHosts() { + String output = client.parseAndGenerateOutput("{\"status\":\"FINISHED\"}"); + assertEquals("File distribution finished", output); + } + + @Test + public void unknownForAllHosts() { + String output = client.parseAndGenerateOutput("{\"status\":\"UNKNOWN\", \"message\":\"Something went wrong\"}"); + assertEquals("File distribution status unknown: Something went wrong", output); + } + + @Test + public void manyHostsVariousStates() { + String statusForTwoHosts = createStatusForTwoHosts(); + System.out.println(statusForTwoHosts); + String output = client.parseAndGenerateOutput(statusForTwoHosts); + assertEquals("File distribution in progress:\nlocalhost1: IN_PROGRESS (1 of 2 finished)\nlocalhost2: UNKNOWN (Connection timed out)", output); + } + + private static CommandLineArguments createArguments(String... args) { + return CommandLineArguments.build(args); + } + + private String createStatusForTwoHosts() { + return "{\"status\":\"IN_PROGRESS\"," + + "\"hosts\":[" + createInProgressStatusForHost("localhost1") + "," + createUnknownStatusForHost("localhost2") + "]" + + "}"; + } + + private String createInProgressStatusForHost(String hostname) { + return "{\"hostname\":\"" + hostname + "\"," + + "\"status\":\"IN_PROGRESS\"," + + "\"message\":\"\"," + + "\"fileReferences\":[" + + "{\"1234\":0.2}, {\"abcd\":1.0}]}"; + } + + private String createUnknownStatusForHost(String hostname) { + return "{\"hostname\":\"" + hostname + "\"," + + "\"status\":\"UNKNOWN\"," + + "\"message\":\"Connection timed out\"}"; + } + +} |