diff options
Diffstat (limited to 'filedistribution')
5 files changed, 0 insertions, 433 deletions
diff --git a/filedistribution/CMakeLists.txt b/filedistribution/CMakeLists.txt index ff6e3b28473..73c581c0b7f 100644 --- a/filedistribution/CMakeLists.txt +++ b/filedistribution/CMakeLists.txt @@ -1,4 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_jar(filedistribution-jar-with-dependencies.jar) - -vespa_install_script(src/main/sh/vespa-status-filedistribution.sh vespa-status-filedistribution bin) diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index e16f8324da9..fb851d8325d 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -20,18 +20,6 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>container-apache-http-client-bundle</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -68,26 +56,6 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> </dependency> - <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-nop</artifactId> - <scope>compile</scope> - <version>${slf4j.version}</version> - </dependency> </dependencies> <build> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java deleted file mode 100644 index a9bbfafeaf0..00000000000 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.filedistribution.status; - -import ai.vespa.util.http.hc5.VespaHttpClientBuilder; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airlift.airline.Command; -import io.airlift.airline.HelpOption; -import io.airlift.airline.Option; -import io.airlift.airline.SingleCommand; -import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.core5.http.HttpEntity; -import org.apache.hc.core5.http.ParseException; -import org.apache.hc.core5.http.io.entity.EntityUtils; -import org.apache.hc.core5.net.URIBuilder; -import org.apache.hc.core5.util.Timeout; - -import javax.inject.Inject; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Map; - -import static org.apache.hc.client5.http.config.RequestConfig.custom; - -/** - * Tool for getting file distribution status - * - * @author hmusum - */ -public class FileDistributionStatusClient { - - private static final String statusUnknown = "UNKNOWN"; - private static final String statusInProgress = "IN_PROGRESS"; - private static final String statusFinished = "FINISHED"; - - private final String tenantName; - private final String applicationName; - private final String instanceName; - private final String environment; - private final String region; - private final double timeout; - private final boolean debug; - - FileDistributionStatusClient(CommandLineArguments arguments) { - tenantName = arguments.getTenantName(); - applicationName = arguments.getApplicationName(); - instanceName = arguments.getInstanceName(); - environment = arguments.getEnvironment(); - region = arguments.getRegion(); - timeout = arguments.getTimeout(); - debug = arguments.getDebugFlag(); - } - - public static void main(String[] args) { - try { - new FileDistributionStatusClient(CommandLineArguments.build(args)).run(); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.exit(1); - } - } - - public void run() { - String json = doHttpRequest(); - System.out.println(parseAndGenerateOutput(json)); - } - - private String doHttpRequest() { - Timeout timeoutInMillis = Timeout.ofMilliseconds((long) (timeout * 1000)); - RequestConfig config = custom() - .setConnectTimeout(timeoutInMillis) - .setConnectionRequestTimeout(timeoutInMillis) - .setResponseTimeout(timeoutInMillis) - .build(); - CloseableHttpClient httpClient = VespaHttpClientBuilder.create().build(); - URI statusUri = createStatusApiUri(); - if (debug) - System.out.println("URI:" + statusUri); - try { - HttpGet request = new HttpGet(statusUri); - request.addHeader("Connection", "Close"); - request.setConfig(config); - CloseableHttpResponse response = httpClient.execute(request); - HttpEntity entity = response.getEntity(); - String content = EntityUtils.toString(entity); - if (debug) - System.out.println("response:" + content); - if (response.getCode() == 200) { - return content; - } else { - throw new RuntimeException("Failed to get status for request " + statusUri + ": " + - response.getCode() + ": " + content); - } - } catch (IOException | ParseException e) { - throw new RuntimeException(e); - } - } - - String parseAndGenerateOutput(String json) { - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode; - try { - jsonNode = objectMapper.readTree(json); - } catch (IOException e) { - throw new RuntimeException(e); - } - String status = jsonNode.get("status").asText(); - switch (status) { - case statusUnknown: - return "File distribution status unknown: " + jsonNode.get("message").asText(); - case statusInProgress: - return "File distribution in progress:\n" + inProgressOutput(jsonNode.get("hosts")); - case statusFinished: - return "File distribution finished"; - default: - throw new RuntimeException("Unknown status " + status); - } - } - - private URI createStatusApiUri() { - String path = String.format("/application/v2/tenant/%s/application/%s/environment/%s/region/%s/instance/%s/filedistributionstatus", - tenantName, applicationName, environment, region, instanceName); - try { - return new URIBuilder() - .setScheme("http") - .setHost("localhost") - .setPort(19071) - .setPath(path) - .addParameter("timeout", String.valueOf(timeout)) - .build(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - - private String inProgressOutput(JsonNode hosts) { - ArrayList<String> statusPerHost = new ArrayList<>(); - for (JsonNode host : hosts) { - String status = host.get("status").asText(); - StringBuilder sb = new StringBuilder(host.get("hostname").asText()).append(": ").append(status); - switch (status) { - case statusUnknown: - sb.append(" (").append(host.get("message").asText()).append(")"); - break; - case statusInProgress: - JsonNode fileReferencesArray = host.get("fileReferences"); - int finished = 0; - for (JsonNode element : fileReferencesArray) { - for (Iterator<Map.Entry<String, JsonNode>> it = element.fields(); it.hasNext(); ) { - Map.Entry<String, JsonNode> fileReferenceStatus = it.next(); - if (fileReferenceStatus.getValue().asDouble() == 1.0) - finished++; - } - } - sb.append(" (" + finished + " of " + fileReferencesArray.size() + " finished)"); - break; - case statusFinished: - break; // Nothing to add - default: - throw new RuntimeException("Unknown status " + status); - } - statusPerHost.add(sb.toString()); - } - return String.join("\n", statusPerHost); - } - - @Command(name = "vespa-status-filedistribution", description = "Tool for getting file distribution status.") - public static class CommandLineArguments { - - static CommandLineArguments build(String[] args) { - CommandLineArguments arguments = null; - try { - arguments = SingleCommand.singleCommand(CommandLineArguments.class).parse(args); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println("Use --help to show usage.\n"); - System.exit(1); - } - - if (arguments.helpOption.showHelpIfRequested()) { - System.exit(0); - } - - if (arguments.getTenantName() == null) { - System.err.println("'--tenant' not set."); - System.exit(1); - } - - if (arguments.getApplicationName() == null) { - System.err.println("'--application' not set."); - System.exit(1); - } - - return arguments; - } - - @Inject - HelpOption helpOption; - - @Option(name = {"--tenant"}, - description = "tenant name") - private String tenantNameArg; - - @Option(name = {"--application"}, - description = "application name") - private String applicationNameArg; - - @Option(name = {"--instance"}, - description = "instance name") - private String instanceNameArg = "default"; - - @Option(name = {"--environment"}, - description = "environment name") - private String environmentArg = "prod"; - - @Option(name = {"--region"}, - description = "region name") - private String regionArg = "default"; - - @Option(name = {"--timeout"}, - description = "The timeout (in seconds).") - private double timeoutArg = 5; - - @Option(name = {"--debug"}, - description = "Print debug log.") - private boolean debugArg; - - public String getTenantName() { return tenantNameArg; } - - public String getApplicationName() { return applicationNameArg; } - - public String getInstanceName() { return instanceNameArg; } - - public String getEnvironment() { return environmentArg; } - - public String getRegion() { return regionArg; } - - public double getTimeout() { return timeoutArg; } - - public boolean getDebugFlag() { return debugArg; } - } - -} diff --git a/filedistribution/src/main/sh/vespa-status-filedistribution.sh b/filedistribution/src/main/sh/vespa-status-filedistribution.sh deleted file mode 100644 index 3d930c72ccf..00000000000 --- a/filedistribution/src/main/sh/vespa-status-filedistribution.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - empty_if_start_slash=${mypath%%/*} - if [ "${empty_if_start_slash}" ]; then - mypath=$(pwd)/${mypath} - fi - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findhost () { - if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 - fi - validate="${VESPA_HOME}/bin/vespa-validate-hostname" - if [ -f "$validate" ]; then - "$validate" "${VESPA_HOSTNAME}" || exit 1 - fi - export VESPA_HOSTNAME -} - -findroot -findhost - -ROOT=${VESPA_HOME%/} -export ROOT - -# END environment bootstrap section - -if [ "$VESPA_ENVIRONMENT" != "" ]; then - environment="--environment $VESPA_ENVIRONMENT" -fi -if [ "$VESPA_REGION" != "" ]; then - region="--region $VESPA_REGION" -fi - -defaults="--tenant default --application default --instance default" -jvmoptions="-XX:MaxJavaStackTraceDepth=1000000 $(getJavaOptionsIPV46) -Xms48m -Xmx48m" -jar="-cp ${VESPA_HOME}/lib/jars/filedistribution-jar-with-dependencies.jar" - -exec java $jvmoptions $jar com.yahoo.vespa.filedistribution.status.FileDistributionStatusClient $defaults $environment $region "$@" diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java deleted file mode 100644 index a724838e7c7..00000000000 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.filedistribution.status; - -import org.junit.Test; - -import static com.yahoo.vespa.filedistribution.status.FileDistributionStatusClient.CommandLineArguments; -import static org.junit.Assert.assertEquals; - -public class FileDistributionStatusClientTest { - - private static final CommandLineArguments arguments = createArguments("--tenant", "foo", "--application", "bar"); - private final FileDistributionStatusClient client = new FileDistributionStatusClient(arguments); - - @Test - public void finishedForAllHosts() { - String output = client.parseAndGenerateOutput("{\"status\":\"FINISHED\"}"); - assertEquals("File distribution finished", output); - } - - @Test - public void unknownForAllHosts() { - String output = client.parseAndGenerateOutput("{\"status\":\"UNKNOWN\", \"message\":\"Something went wrong\"}"); - assertEquals("File distribution status unknown: Something went wrong", output); - } - - @Test - public void manyHostsVariousStates() { - String statusForTwoHosts = createStatusForTwoHosts(); - System.out.println(statusForTwoHosts); - String output = client.parseAndGenerateOutput(statusForTwoHosts); - assertEquals("File distribution in progress:\nlocalhost1: IN_PROGRESS (1 of 2 finished)\nlocalhost2: UNKNOWN (Connection timed out)", output); - } - - private static CommandLineArguments createArguments(String... args) { - return CommandLineArguments.build(args); - } - - private String createStatusForTwoHosts() { - return "{\"status\":\"IN_PROGRESS\"," + - "\"hosts\":[" + createInProgressStatusForHost("localhost1") + "," + createUnknownStatusForHost("localhost2") + "]" + - "}"; - } - - private String createInProgressStatusForHost(String hostname) { - return "{\"hostname\":\"" + hostname + "\"," + - "\"status\":\"IN_PROGRESS\"," + - "\"message\":\"\"," + - "\"fileReferences\":[" + - "{\"1234\":0.2}, {\"abcd\":1.0}]}"; - } - - private String createUnknownStatusForHost(String hostname) { - return "{\"hostname\":\"" + hostname + "\"," + - "\"status\":\"UNKNOWN\"," + - "\"message\":\"Connection timed out\"}"; - } - -} |