diff options
71 files changed, 715 insertions, 309 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java index 31e1a8519f4..e4e964c7088 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslKeyStoreConfigurator.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; @@ -9,25 +9,19 @@ import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.jdisc.http.ssl.SslKeyStoreConfigurator; import com.yahoo.jdisc.http.ssl.SslKeyStoreContext; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; +import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.AthenzCertificateClient; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.Path; import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.PrivateKey; -import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; @@ -37,6 +31,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import static com.yahoo.vespa.athenz.tls.KeyStoreUtils.writeKeyStoreToFile; import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; /** @@ -87,15 +82,14 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements private static Optional<KeyStore> tryReadKeystoreFile(File certificateFile, Duration updatePeriod) { try { if (!certificateFile.exists()) return Optional.empty(); - KeyStore keyStore = KeyStore.getInstance("JKS"); - try (InputStream in = new BufferedInputStream(new FileInputStream(certificateFile))) { - keyStore.load(in, new char[0]); - } + KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) + .fromFile(certificateFile) + .build(); Instant minimumExpiration = Instant.now().plus(updatePeriod).plus(EXPIRATION_MARGIN); boolean isExpired = getCertificateExpiry(keyStore).isBefore(minimumExpiration); if (isExpired) return Optional.empty(); return Optional.of(keyStore); - } catch (IOException | GeneralSecurityException e) { + } catch (GeneralSecurityException e) { log.log(LogLevel.ERROR, "Failed to read keystore from disk: " + e.getMessage(), e); return Optional.empty(); } @@ -139,28 +133,23 @@ public class AthenzSslKeyStoreConfigurator extends AbstractComponent implements AthenzCertificateClient certificateClient, AthenzProviderServiceConfig.Zones zoneConfig, Path keystoreCachePath) { - try { - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - X509Certificate certificate = certificateClient.updateCertificate(privateKey); - Instant expirationTime = certificate.getNotAfter().toInstant(); - Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); - log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); - - KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(null); - keyStore.setKeyEntry( - CERTIFICATE_ALIAS, privateKey, CERTIFICATE_PASSWORD.toCharArray(), new Certificate[]{certificate}); - tryWriteKeystore(keyStore, keystoreCachePath); - return keyStore; - } catch (IOException | GeneralSecurityException e) { - throw new RuntimeException(e); - } + PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); + X509Certificate certificate = certificateClient.updateCertificate(privateKey); + Instant expirationTime = certificate.getNotAfter().toInstant(); + Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); + log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); + + KeyStore keyStore = KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry(CERTIFICATE_ALIAS, privateKey, CERTIFICATE_PASSWORD.toCharArray(), certificate) + .build(); + tryWriteKeystore(keyStore, keystoreCachePath); + return keyStore; } private static void tryWriteKeystore(KeyStore keyStore, Path keystoreCachePath) { - try (OutputStream out = new BufferedOutputStream(new FileOutputStream(keystoreCachePath.toFile()))) { - keyStore.store(out, new char[0]); - } catch (IOException | GeneralSecurityException e) { + try { + writeKeyStoreToFile(keyStore, keystoreCachePath.toFile()); + } catch (Exception e) { log.log(LogLevel.ERROR, "Failed to write keystore to disk: " + e.getMessage(), e); } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java index 7e24109a197..376dd2ed4ac 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzSslTrustStoreConfigurator.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; @@ -6,6 +6,8 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.jdisc.http.ssl.SslTrustStoreConfigurator; import com.yahoo.jdisc.http.ssl.SslTrustStoreContext; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; +import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; @@ -20,7 +22,7 @@ import org.bouncycastle.operator.ContentSigner; import org.bouncycastle.operator.OperatorCreationException; import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; -import java.io.FileInputStream; +import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.security.KeyPair; @@ -70,12 +72,10 @@ public class AthenzSslTrustStoreConfigurator implements SslTrustStoreConfigurato KeyPair keyPair = getKeyPair(keyProvider, configserverConfig, athenzProviderServiceConfig); X509Certificate selfSignedCertificate = createSelfSignedCertificate(keyPair, configserverConfig); log.log(LogLevel.FINE, "Generated self-signed certificate: " + selfSignedCertificate); - KeyStore trustStore = KeyStore.getInstance("JKS"); - try (FileInputStream in = new FileInputStream(athenzProviderServiceConfig.athenzCaTrustStore())) { - trustStore.load(in, "changeit".toCharArray()); - } - trustStore.setCertificateEntry(CERTIFICATE_ALIAS, selfSignedCertificate); - return trustStore; + return KeyStoreBuilder.withType(KeyStoreType.JKS) + .fromFile(new File(athenzProviderServiceConfig.athenzCaTrustStore()), "changeit".toCharArray()) + .withCertificateEntry(CERTIFICATE_ALIAS, selfSignedCertificate) + .build(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index bca3fbdc960..69790dfc857 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -179,6 +179,7 @@ public class DeploymentJobs { productionApNortheast2 ("production-ap-northeast-2" , ZoneId.from("prod" , "ap-northeast-2") , null ), productionApSoutheast1 ("production-ap-southeast-1" , ZoneId.from("prod" , "ap-southeast-1") , null ), productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ), + productionAwsUsEast1a ("production-aws-us-east-1a" , ZoneId.from("prod" , "aws-us-east-1a") , null ), productionCdUsCentral1 ("production-cd-us-central-1", null , ZoneId.from("prod" , "cd-us-central-1")), productionCdUsCentral2 ("production-cd-us-central-2", null , ZoneId.from("prod" , "cd-us-central-2")); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java index 261c2c0f2ad..ca939a9f862 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java @@ -13,7 +13,7 @@ import java.time.Duration; import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; -import static com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder.KeyStoreType.JKS; +import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; /** * @author bjorncs diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 805b1e69d45..4beb6eea055 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -164,6 +164,8 @@ public class DockerImpl implements Docker { .withPassword(credentials.password)) .ifPresent(pullImageCmd::withAuthConfig); + logger.log(LogLevel.INFO, "Starting download of " + image.asString()); + pullImageCmd.exec(new ImagePullCallback(image)); return true; } @@ -467,6 +469,7 @@ public class DockerImpl implements Docker { public void onComplete() { Optional<InspectImageResponse> image = inspectImage(dockerImage); if (image.isPresent()) { // Download successful, update image GC with the newly downloaded image + logger.log(LogLevel.INFO, "Download completed: " + dockerImage.asString()); dockerImageGC.ifPresent(imageGC -> imageGC.updateLastUsedTimeFor(image.get().getId())); removeScheduledPoll(dockerImage); } else { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java index da7be1bff6d..4fb3049c39d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java @@ -58,6 +58,7 @@ public class Environment { private final Optional<KeyStoreOptions> trustStoreOptions; private final Optional<AthenzIdentity> athenzIdentity; private final NodeType nodeType; + private final String defaultFlavor; static { filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -71,7 +72,8 @@ public class Environment { new PathResolver(), Optional.of(getEnvironmentVariable(COREDUMP_FEED_ENDPOINT)), - NodeType.host); + NodeType.host, + "d-2-8-50"); } public Environment(ConfigServerConfig configServerConfig, @@ -80,7 +82,8 @@ public class Environment { String hostedSystem, PathResolver pathResolver, Optional<String> coreDumpFeedEndpoint, - NodeType nodeType) { + NodeType nodeType, + String defaultFlavor) { this(configServerConfig, hostedEnvironment, hostedRegion, @@ -90,7 +93,8 @@ public class Environment { pathResolver, getLogstashNodesFromEnvironment(), coreDumpFeedEndpoint, - nodeType + nodeType, + defaultFlavor ); } @@ -103,7 +107,8 @@ public class Environment { PathResolver pathResolver, List<String> logstashNodes, Optional<String> feedEndpoint, - NodeType nodeType) { + NodeType nodeType, + String defaultFlavor) { this.configServerHostNames = configServerConfig.hosts(); this.configServerURIs = createConfigServerUris( configServerConfig.scheme(), @@ -132,6 +137,7 @@ public class Environment { this.logstashNodes = logstashNodes; this.feedEndpoint = feedEndpoint; this.nodeType = nodeType; + this.defaultFlavor = defaultFlavor; } public List<String> getConfigServerHostNames() { return configServerHostNames; } @@ -272,6 +278,8 @@ public class Environment { public NodeType getNodeType() { return nodeType; } + public String getDefaultFlavor() { return defaultFlavor; } + public static class Builder { private ConfigServerConfig configServerConfig; private String environment; @@ -283,6 +291,7 @@ public class Environment { private List<String> logstashNodes = Collections.emptyList(); private Optional<String> feedEndpoint = Optional.empty(); private NodeType nodeType = NodeType.tenant; + private String defaultFlavor; public Builder configServerConfig(ConfigServerConfig configServerConfig) { this.configServerConfig = configServerConfig; @@ -334,11 +343,17 @@ public class Environment { return this; } + public Builder defaultFlavor(String defaultFlavor) { + this.defaultFlavor = defaultFlavor; + return this; + } + public Environment build() { Objects.requireNonNull(configServerConfig, "configServerConfig cannot be null"); Objects.requireNonNull(environment, "environment cannot be null"); Objects.requireNonNull(region, "region cannot be null"); Objects.requireNonNull(system, "system cannot be null"); + Objects.requireNonNull(defaultFlavor, "default flavor cannot be null"); return new Environment(configServerConfig, environment, @@ -349,7 +364,8 @@ public class Environment { Optional.ofNullable(pathResolver).orElseGet(PathResolver::new), logstashNodes, feedEndpoint, - nodeType); + nodeType, + defaultFlavor); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java index 7abe9bce718..04b222875c3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConfigServerApiImpl.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder; -import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder.KeyStoreType; +import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java index a69fac508df..d77e468bc8d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ConfigServerContainerData.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.node.admin.containerdata; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.Environment; import java.nio.file.Path; @@ -18,9 +17,7 @@ public class ConfigServerContainerData { this.configServerNodeHostName = configServerNodeHostName; } - public void create() { - ContainerData containerData = ContainerData.createCleanContainerData( - environment, ContainerName.fromHostname(configServerNodeHostName)); + public void writeTo(ContainerData containerData) { containerData.addFile(getPath("configserver-config.xml"), createConfigServerConfigXml()); containerData.addFile(getPath("node-repository-config.xml"), createNodeRepoConfigXml()); } @@ -37,8 +34,7 @@ public class ConfigServerContainerData { " <hostedVespa>true</hostedVespa>\n" + " <multitenant>true</multitenant>\n" + " <useVespaVersionInRequest>true</useVespaVersionInRequest>\n" + - // TODO: Avoid hardcoding of default flavor - " <defaultFlavor>t2.xlarge</defaultFlavor>\n" + + " <defaultFlavor>" + environment.getDefaultFlavor() + "</defaultFlavor>\n" + " <zookeeper>\n" + " <barrierTimeout>1200</barrierTimeout>\n" + " </zookeeper>\n" + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java index ebfb433de4a..1dc06d52e87 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/ContainerData.java @@ -37,7 +37,7 @@ public class ContainerData { this.destinationPathOnHost = environment.pathInHostFromPathInNode(containerName, ContainerData.containerDataPath); } - public static ContainerData createCleanContainerData(Environment environment, ContainerName containerName) { + public static ContainerData createClean(Environment environment, ContainerName containerName) { ContainerData containerData = new ContainerData(environment, containerName); IOExceptionUtil.uncheck(containerData::cleanup); return containerData; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/MotdContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/MotdContainerData.java new file mode 100644 index 00000000000..24dadd41b5a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/MotdContainerData.java @@ -0,0 +1,71 @@ +package com.yahoo.vespa.hosted.node.admin.containerdata; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.task.util.file.Template; +import com.yahoo.vespa.hosted.provision.Node.State; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.function.BiConsumer; + +/** + * @author jvenstad + */ +public class MotdContainerData { + + private static final Path motdPath = Paths.get("etc/profile.d/motd.sh"); + private static final String templateString = "#!/usr/bin/env bash\n" + + "\n" + + "function motd {\n" + + "\n" + + " local -r uptime=$(uptime | cut -f 3- -d ' ')\n" + + "\n" + + " local -r no_color='\\e[0m'\n" + + " local -r green='\\e[0;32m'\n" + + "## Use red zone name for main prod zones, yellow for other main zones and no colour for cd and dev zones.\n" + + " local -r alert=#if($zone.getSystem() == \"main\")#if($zone.getEnvironment() == \"prod\")'\\e[0;91m'#else'\\e[0;33m'#end#else$green#end\n" + + "\n" + + "\n" + + " echo -e \"\n" + + "${green}Zone : ${alert}$zone.getSystem().toUpperCase() $zone.getEnvironment().toUpperCase() $zone.getRegion().toUpperCase()\n" + + "${green}Node type : ${no_color}$type\n" + + "${green}Node flavor : ${no_color}$flavor\n" + + "${green}Host name : ${no_color}$(hostname)\n" + + "${green}Uptime : ${no_color}$uptime\n" + + "${green}Version : ${no_color}wanted = $wanted.orElse(\"unknown\"); installed = $installed.orElse(\"unknown\")\n" + + "#if($owner.isPresent())\n" + + "${green}Node state : ${no_color}$state\n" + + "${green}Owner : ${no_color}$owner.get().serializedForm()\n" + + "#end\n" + + "\"\n" + + "}\n" + + "\n" + + "# Display motd (gently)\n" + + "[ ! -f ~/.hushlogin ] && motd\n"; + + private final String renderedString; + + public MotdContainerData(ContainerNodeSpec nodeSpec, Environment environment) { + renderedString = Template.of(templateString) + .set("zone", environment) + .set("type", nodeSpec.nodeType) + .set("state", nodeSpec.nodeState) + .set("installed", nodeSpec.vespaVersion) + .set("wanted", nodeSpec.wantedVespaVersion) + .set("owner", nodeSpec.owner.map(id -> ApplicationId.from(id.tenant, id.application, id.instance))) + .set("flavor", nodeSpec.nodeFlavor) + .render(); + } + + public void writeTo(ContainerData containerData) { + writeTo(containerData::addFile); + } + + void writeTo(BiConsumer<Path, String> fileWriter) { + fileWriter.accept(motdPath, renderedString); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java new file mode 100644 index 00000000000..ea9b2312b77 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerData.java @@ -0,0 +1,57 @@ +package com.yahoo.vespa.hosted.node.admin.containerdata; + +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.task.util.file.Template; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.function.BiConsumer; + +public class PromptContainerData { + + private static final Path promptPath = Paths.get("etc/profile.d/prompt.sh"); + private static final String templateString = "# Make sure we get UTC/GMT all over\n" + + "export TZ=UTC\n" + + "\n" + + "# Skip the rest for non-interactice shells\n" + + "[ -z \"$PS1\" ] && return\n" + + "\n" + + "# Check the window size after each command and, if necessary,\n" + + "# Update the values of LINES and COLUMNS.\n" + + "shopt -s checkwinsize\n" + + "\n" + + "# Colors; see https://wiki.archlinux.org/index.php/Color_Bash_Prompt\n" + + "color_off='\\[\\e[0m\\]' # Text Reset\n" + + "color_bold='\\[\\e[1m\\]' # Bold text\n" + + "\n" + + "env_colour=#if($zone.getSystem() == \"main\")#if($zone.getEnvironment() == \"prod\")'\\e[0;91m'#else'\\e[0;33m'#end#else$green#end\n" + + "\n" + + "\n" + + "PS1=\"${env_colour}$zone.getRegion().toUpperCase()${color_off} [\\u@${color_bold}\\h${color_off}:\\w]\\$ \"\n" + + "\n" + + "# Fix colors\n" + + "if type dircolors > /dev/null 2>&1; then\n" + + " eval $(dircolors -b)\n" + + "fi\n" + + "\n" + + "# Make PS1 available in sub-shells\n" + + "export PS1\n" + + "\n"; + + private final String renderedString; + + public PromptContainerData(Environment environment) { + renderedString = Template.of(templateString) + .set("zone", environment) + .render(); + } + + public void writeTo(ContainerData containerData) { + writeTo(containerData::addFile); + } + + void writeTo(BiConsumer<Path, String> fileWriter) { + fileWriter.accept(promptPath, renderedString); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index fb31a7e5571..26a2d10f229 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -359,6 +359,7 @@ public class DockerOperationsImpl implements DockerOperations { */ private static Map<Path, Boolean> getDirectoriesToMount(Environment environment) { final Map<Path, Boolean> directoriesToMount = new HashMap<>(); + directoriesToMount.put(Paths.get("/var/log/secret-agent"), false); directoriesToMount.put(Paths.get("/etc/yamas-agent"), true); directoriesToMount.put(Paths.get("/etc/filebeat"), true); directoriesToMount.put(environment.pathInNodeUnderVespaHome("logs/daemontools_y"), false); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index a09dea69a1c..72d488e361f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -98,6 +98,7 @@ public class StorageMaintainer { vespaSchedule.writeTo(yamasAgentFolder); hostLifeSchedule.writeTo(yamasAgentFolder); final String[] restartYamasAgent = new String[]{"service", "yamas-agent", "restart"}; + writeSecretAgentConfig(containerName); dockerOperations.executeCommandInContainerAsRoot(containerName, restartYamasAgent); } catch (IOException e) { throw new RuntimeException("Failed to write secret-agent schedules for " + containerName, e); @@ -430,4 +431,33 @@ public class StorageMaintainer { nextHandleOldCoredumpsAt = Instant.EPOCH; } } + + private void writeSecretAgentConfig(ContainerName containerName) { + String config = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + + "<log4j:configuration xmlns:log4j=\"http://jakarta.apache.org/log4j/\">\n" + + " <appender name=\"AppLog\" class=\"org.apache.log4j.rolling.RollingFileAppender\">\n" + + " <param name=\"Threshold\" value=\"Debug\" />\n" + + " <rollingPolicy class=\"org.apache.log4j.rolling.FixedWindowRollingPolicy\">\n" + + " <param name=\"FileNamePattern\" value=\"/var/log/secret-agent/collector-writer.%i\"/>\n" + + " <param name=\"MinIndex\" value=\"0\"/>\n" + + " <param name=\"MaxIndex\" value=\"100\"/>\n" + + " </rollingPolicy>\n" + + " <triggeringPolicy class=\"org.apache.log4j.rolling.SizeBasedTriggeringPolicy\">\n" + + " <param name=\"maxFileSize\" value=\"10MB\"/>\n" + + " </triggeringPolicy>\n" + + " <layout class=\"org.apache.log4j.PatternLayout\">\n" + + " <param name=\"ConversionPattern\" value=\"%d %-5p [%c] - %m%n\" />\n" + + " </layout>\n" + + " </appender>\n" + + "\n" + + " <root>\n" + + " <priority value=\"all\" />\n" + + " <appender-ref ref=\"AppLog\" />\n" + + " </root>\n" + + "</log4j:configuration>"; + + Path configPath = Paths.get("/etc/secret-agent/log4cxx.collector-writer.xml"); + String writeConfigCommand = String.format("echo '%s' > %s", config, configPath); + dockerOperations.executeCommandInContainerAsRoot(containerName, "/bin/sh", "-c", writeConfigCommand); + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 37464dcb50b..27e7cbff68e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -16,6 +16,9 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.containerdata.ContainerData; +import com.yahoo.vespa.hosted.node.admin.containerdata.MotdContainerData; +import com.yahoo.vespa.hosted.node.admin.containerdata.PromptContainerData; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.containerdata.ConfigServerContainerData; @@ -681,9 +684,17 @@ public class NodeAgentImpl implements NodeAgent { } private void createContainerData(ContainerNodeSpec nodeSpec) { + ContainerData containerData = ContainerData.createClean(environment, + ContainerName.fromHostname(nodeSpec.hostname)); + if (nodeSpec.nodeType.equals(NodeType.config.name())) { logger.info("Creating files needed by config server"); - new ConfigServerContainerData(environment, nodeSpec.hostname).create(); + new ConfigServerContainerData(environment, nodeSpec.hostname).writeTo(containerData); } + + logger.info("Creating files for message of the day and the bash prompt"); + new MotdContainerData(nodeSpec, environment).writeTo(containerData); + new PromptContainerData(environment).writeTo(containerData); } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java new file mode 100644 index 00000000000..6092cc1f038 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Template.java @@ -0,0 +1,55 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; + +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Uses the Velocity engine to render a template, to and from both String and Path objects. + * + * @author hakonhall + * @author jvenstad + */ +public class Template { + + private final VelocityEngine velocityEngine = new VelocityEngine(); + private final VelocityContext velocityContext = new VelocityContext(); + private final String template; + + private Template(String template) { + this.template = template; + + velocityEngine.addProperty(Velocity.RUNTIME_LOG_LOGSYSTEM_CLASS, + "org.apache.velocity.runtime.log.NullLogSystem"); + velocityEngine.init(); + } + + public static Template at(Path templatePath) { + return of(IOExceptionUtil.uncheck(() -> new String(Files.readAllBytes(templatePath)))); + } + + public static Template of(String template) { + return new Template(template); + } + + public Template set(String name, Object value) { + velocityContext.put(name, value); + return this; + } + + public FileWriter getFileWriterTo(Path destinationPath) { + return new FileWriter(destinationPath, this::render); + } + + public String render() { + StringWriter writer = new StringWriter(); + velocityEngine.evaluate(velocityContext, writer, "Template", template); + return writer.toString(); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java deleted file mode 100644 index e9d0770f933..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFile.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.apache.velocity.Template; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; -import org.apache.velocity.app.VelocityEngine; - -import java.io.StringWriter; -import java.nio.file.Path; - -/** - * Make a file based on a Velocity template file. - * - * @author hakonhall - */ -public class TemplateFile { - private final Path templatePath; - private final VelocityEngine velocityEngine; - private final VelocityContext velocityContext = new VelocityContext(); - - public TemplateFile(Path templatePath) { - this.templatePath = templatePath; - velocityEngine = new VelocityEngine(); - velocityEngine.addProperty( - Velocity.RUNTIME_LOG_LOGSYSTEM_CLASS, - "org.apache.velocity.runtime.log.NullLogSystem"); - velocityEngine.addProperty(Velocity.FILE_RESOURCE_LOADER_PATH, templatePath.getParent().toString()); - velocityEngine.init(); - } - - public TemplateFile set(String name, Object value) { - velocityContext.put(name, value); - return this; - } - - public FileWriter getFileWriterTo(Path destinationPath) { - return new FileWriter(destinationPath, this::render); - } - - private String render() { - Template template = velocityEngine.getTemplate(templatePath.getFileName().toString(), "UTF-8"); - StringWriter writer = new StringWriter(); - template.merge(velocityContext, writer); - return writer.toString(); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/containerdata/MotdContainerDataTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/containerdata/MotdContainerDataTest.java new file mode 100644 index 00000000000..fe095cd0002 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/containerdata/MotdContainerDataTest.java @@ -0,0 +1,61 @@ +package com.yahoo.vespa.hosted.node.admin.containerdata; + +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MotdContainerDataTest { + + @Test + public void writesMotd() { + MotdContainerData motdContainerData = new MotdContainerData(new ContainerNodeSpec.Builder() + .nodeType("tenant") + .owner(new ContainerNodeSpec.Owner("tenant1", + "application1", + "default")) + .nodeState(Node.State.dirty) + .vespaVersion("7.0.0") + .hostname("nope") + .nodeFlavor("D-WAVE") + .allowedToBeDown(false) + .membership(new ContainerNodeSpec.Membership(null, null, null, 0, false)) + .minCpuCores(0) + .minMainMemoryAvailableGb(0) + .minDiskAvailableGb(0) + .fastDisk(false) + .ipAddresses(Collections.emptySet()) + .build(), + new Environment.Builder() + .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) + .system("main") + .environment("prod") + .region("aws-us-east-1a") + .defaultFlavor("cherry") + .build()); + + motdContainerData.writeTo((path, content) -> { + assertEquals(path, Paths.get("etc/profile.d/motd.sh")); + + System.out.println(content); + + assertTrue(content.contains("tenant")); + assertTrue(content.contains("D-WAVE")); + assertTrue(content.contains("[0;91m")); + assertTrue(content.contains("MAIN PROD AWS-US-EAST-1A")); + assertTrue(content.contains("tenant1:application1:default")); + assertTrue(content.contains("dirty")); + assertTrue(content.contains("wanted = unknown")); + assertTrue(content.contains("installed = 7.0.0")); + }); + + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerDataTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerDataTest.java new file mode 100644 index 00000000000..4403b5bdad0 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/containerdata/PromptContainerDataTest.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.node.admin.containerdata; + +import com.yahoo.vespa.hosted.node.admin.component.Environment; +import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; +import org.junit.Test; + +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class PromptContainerDataTest { + + @Test + public void writesPrompt() { + PromptContainerData promptContainerData = new PromptContainerData(new Environment.Builder() + .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) + .system("main") + .environment("prod") + .region("aws-us-east-1a") + .defaultFlavor("cherry") + .build()); + + promptContainerData.writeTo((path, content) -> { + assertEquals(path, Paths.get("etc/profile.d/prompt.sh")); + + assertTrue(content.contains("[0;91m")); + assertTrue(content.contains("AWS-US-EAST-1A")); + }); + + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index b0fc877bd88..81df3e52a21 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -34,6 +34,7 @@ public class DockerOperationsImplTest { .region("us-east-1") .environment("prod") .system("main") + .defaultFlavor("d-2-8-50") .build(); private final Docker docker = mock(Docker.class); private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 7923a58ec47..daf87422b9d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -66,6 +66,7 @@ public class DockerTester implements AutoCloseable { .environment("prod") .system("main") .pathResolver(new PathResolver(pathToVespaHome, Paths.get("/tmp"), Paths.get("/tmp"))) + .defaultFlavor("d-2-8-50") .build(); Clock clock = Clock.systemUTC(); DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, null, new IPAddressesImpl()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java index 9de88cbb8b3..d4eb327439a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -102,6 +102,7 @@ public class FilebeatConfigProviderTest { .region(region) .system(system) .logstashNodes(logstashNodes) + .defaultFlavor("d-2-8-50") .build(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index cb7c22f1da9..9b618b9cf45 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -40,6 +40,7 @@ public class StorageMaintainerTest { .environment("prod") .system("main") .pathResolver(new PathResolver()) + .defaultFlavor("d-2-8-50") .build(); private final DockerOperations docker = mock(DockerOperations.class); private final ProcessExecuter processExecuter = mock(ProcessExecuter.class); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 6a93d4e3664..9d379babeca 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -88,7 +88,9 @@ public class NodeAgentImplTest { .system("main") .parentHostHostname("parent.host.name.yahoo.com") .inetAddressResolver(new InetAddressResolver()) - .pathResolver(pathResolver).build(); + .pathResolver(pathResolver) + .defaultFlavor("d-2-8-50") + .build(); private final ContainerNodeSpec.Builder nodeSpecBuilder = new ContainerNodeSpec.Builder() .hostname(hostName) @@ -173,6 +175,7 @@ public class NodeAgentImplTest { when(nodeRepository.getContainerNodeSpec(hostName)).thenReturn(Optional.of(nodeSpec)); when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(201326592000L)); @@ -228,7 +231,7 @@ public class NodeAgentImplTest { } @Test - public void containerIsRestartedIfFlavorChanged() { + public void containerIsRestartedIfFlavorChanged() throws IOException { final long wantedRestartGeneration = 1; final long currentRestartGeneration = 1; ContainerNodeSpec.Builder specBuilder = nodeSpecBuilder @@ -251,6 +254,7 @@ public class NodeAgentImplTest { .thenReturn(Optional.of(thirdSpec)); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(201326592000L)); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); nodeAgent.converge(); nodeAgent.converge(); @@ -470,6 +474,7 @@ public class NodeAgentImplTest { when(nodeRepository.getContainerNodeSpec(eq(hostName))).thenReturn(Optional.of(nodeSpec)); when(pathResolver.getApplicationStoragePathForNodeAdmin()).thenReturn(Files.createTempDirectory("foo")); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(201326592000L)); nodeAgent.tick(); @@ -581,6 +586,7 @@ public class NodeAgentImplTest { when(dockerOperations.getContainerStats(eq(containerName))) .thenReturn(Optional.of(stats1)) .thenReturn(Optional.of(stats2)); + when(pathResolver.getApplicationStoragePathForHost()).thenReturn(Files.createTempDirectory("bar")); nodeAgent.converge(); // Run the converge loop once to initialize lastNodeSpec nodeAgent.updateContainerNodeMetrics(); // Update metrics once to init and lastCpuMetric diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java index b1d88fdaaee..7aa672e9e80 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateFileTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplateTest.java @@ -2,40 +2,31 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; +import com.google.common.jimfs.Jimfs; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import org.junit.Rule; +import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import java.io.IOException; +import java.nio.file.FileSystem; import java.nio.file.Path; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; -/** - * WARNING: Velocity does not honor an alternative FileSystem like JimFS. - */ -public class TemplateFileTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private void writeFile(Path path, String content) { - UnixPath unixPath = new UnixPath(path); - unixPath.createParents(); - unixPath.writeUtf8File(content); - } +public class TemplateTest { @Test public void basic() throws IOException { + FileSystem fileSystem = TestFileSystem.create(); + Path templatePath = fileSystem.getPath("/example.vm"); String templateContent = "a $x, $y b"; - Path templatePath = folder.newFile("example.vm").toPath(); - writeFile(templatePath, templateContent); + new UnixPath(templatePath).writeUtf8File(templateContent); - Path toPath = folder.newFile().toPath(); + Path toPath = fileSystem.getPath("/example"); TaskContext taskContext = mock(TaskContext.class); - boolean converged = new TemplateFile(templatePath) + boolean converged = Template.at(templatePath) .set("x", "foo") .set("y", "bar") .getFileWriterTo(toPath) @@ -46,4 +37,5 @@ public class TemplateFileTest { String actualContent = new UnixPath(toPath).readUtf8File(); assertEquals("a foo, bar b", actualContent); } + }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java index 80749e5d34b..83edd573260 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/EnvironmentTest.java @@ -23,6 +23,7 @@ public class EnvironmentTest { .environment("prod") .system("main") .pathResolver(new PathResolver()) + .defaultFlavor("d-2-8-50") .build(); @Test diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java index 2bfcaae79e6..6a136c3c111 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/AthenzIdentityProviderImpl.java @@ -17,14 +17,12 @@ import java.io.File; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; -import static com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder.KeyStoreType.JKS; +import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; /** * @author mortent @@ -79,8 +77,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private void registerInstance() { try { credentials = athenzCredentialsService.registerInstance(); - scheduler.scheduleAtFixedRate(this::refreshCertificate, 0, 5, TimeUnit.MINUTES); - scheduler.scheduleAtFixedRate(this::reportMetrics, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); + scheduler.scheduleAtFixedRate(this::refreshCertificate, UPDATE_PERIOD.toMinutes(), UPDATE_PERIOD.toMinutes(), TimeUnit.MINUTES); + scheduler.scheduleAtFixedRate(this::reportMetrics, 0, 5, TimeUnit.MINUTES); } catch (Throwable t) { throw new AthenzIdentityProviderException("Could not retrieve Athenz credentials", t); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilder.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilder.java index fdf58f9e64b..57fc7d1b581 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilder.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.athenz.tls; import com.yahoo.vespa.athenz.api.AthenzIdentityCertificate; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; @@ -10,14 +9,11 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.security.GeneralSecurityException; import java.security.KeyStore; -import java.security.KeyStoreException; import java.security.PrivateKey; -import java.security.cert.Certificate; import java.security.cert.X509Certificate; /** @@ -25,22 +21,6 @@ import java.security.cert.X509Certificate; */ public class AthenzSslContextBuilder { - public enum KeyStoreType { - JKS { - KeyStore createKeystore() throws KeyStoreException { - return KeyStore.getInstance("JKS"); - } - }, - PKCS12 { - private final BouncyCastleProvider bouncyCastleProvider = new BouncyCastleProvider(); - - KeyStore createKeystore() throws KeyStoreException { - return KeyStore.getInstance("PKCS12", bouncyCastleProvider); - } - }; - abstract KeyStore createKeystore() throws GeneralSecurityException; - } - private KeyStoreSupplier trustStoreSupplier; private KeyStoreSupplier keyStoreSupplier; private char[] keyStorePassword; @@ -48,7 +28,7 @@ public class AthenzSslContextBuilder { public AthenzSslContextBuilder() {} public AthenzSslContextBuilder withTrustStore(File file, KeyStoreType trustStoreType) { - this.trustStoreSupplier = () -> loadKeyStoreFromFile(file, null, trustStoreType); + this.trustStoreSupplier = () -> KeyStoreBuilder.withType(trustStoreType).fromFile(file).build(); return this; } @@ -63,7 +43,7 @@ public class AthenzSslContextBuilder { public AthenzSslContextBuilder withKeyStore(PrivateKey privateKey, X509Certificate certificate) { char[] pwd = new char[0]; - this.keyStoreSupplier = () -> createJksKeyStore(privateKey, certificate, pwd); + this.keyStoreSupplier = () -> KeyStoreBuilder.withType(KeyStoreType.JKS).withKeyEntry("athenz", privateKey, certificate).build(); this.keyStorePassword = pwd; return this; } @@ -75,7 +55,7 @@ public class AthenzSslContextBuilder { } public AthenzSslContextBuilder withKeyStore(File file, char[] password, KeyStoreType keyStoreType) { - this.keyStoreSupplier = () -> loadKeyStoreFromFile(file, password, keyStoreType); + this.keyStoreSupplier = () -> KeyStoreBuilder.withType(keyStoreType).fromFile(file, password).build(); this.keyStorePassword = password; return this; } @@ -112,23 +92,6 @@ public class AthenzSslContextBuilder { return keyManagerFactory.getKeyManagers(); } - private static KeyStore loadKeyStoreFromFile(File file, char[] password, KeyStoreType keyStoreType) - throws IOException, GeneralSecurityException{ - KeyStore keyStore = keyStoreType.createKeystore(); - try (FileInputStream in = new FileInputStream(file)) { - keyStore.load(in, password); - } - return keyStore; - } - - private KeyStore createJksKeyStore(PrivateKey privateKey, X509Certificate certificate, char[] password) - throws GeneralSecurityException, IOException{ - KeyStore keyStore = KeyStoreType.JKS.createKeystore(); - keyStore.load(null); - keyStore.setKeyEntry("athenz-identity", privateKey, password, new Certificate[]{certificate}); - return keyStore; - } - private interface KeyStoreSupplier { KeyStore get() throws IOException, GeneralSecurityException; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilder.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilder.java new file mode 100644 index 00000000000..a9279f45129 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilder.java @@ -0,0 +1,121 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.tls; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.singletonList; + +/** + * @author bjorncs + */ +public class KeyStoreBuilder { + + private final List<KeyEntry> keyEntries = new ArrayList<>(); + private final List<CertificateEntry> certificateEntries = new ArrayList<>(); + + private final KeyStoreType keyStoreType; + private File inputFile; + private char[] inputFilePassword; + + private KeyStoreBuilder(KeyStoreType keyStoreType) { + this.keyStoreType = keyStoreType; + } + + public static KeyStoreBuilder withType(KeyStoreType type) { + return new KeyStoreBuilder(type); + } + + public KeyStoreBuilder fromFile(File file, char[] password) { + this.inputFile = file; + this.inputFilePassword = password; + return this; + } + + public KeyStoreBuilder fromFile(File file) { + return fromFile(file, null); + } + + public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, char[] password, List<X509Certificate> certificateChain) { + keyEntries.add(new KeyEntry(alias, privateKey, certificateChain, password)); + return this; + } + + public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, char[] password, X509Certificate certificate) { + return withKeyEntry(alias, privateKey, password, singletonList(certificate)); + } + + public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, X509Certificate certificate) { + return withKeyEntry(alias, privateKey, null, certificate); + } + + public KeyStoreBuilder withKeyEntry(String alias, PrivateKey privateKey, List<X509Certificate> certificateChain) { + return withKeyEntry(alias, privateKey, null, certificateChain); + } + + public KeyStoreBuilder withCertificateEntry(String alias, X509Certificate certificate) { + certificateEntries.add(new CertificateEntry(alias, certificate)); + return this; + } + + public KeyStore build() { + try { + KeyStore keystore = this.keyStoreType.createKeystore(); + if (this.inputFile != null) { + try (InputStream in = new BufferedInputStream(new FileInputStream(this.inputFile))) { + keystore.load(in, this.inputFilePassword); + } + } else { + keystore.load(null); + } + for (KeyEntry entry : keyEntries) { + char[] password = entry.password != null ? entry.password : new char[0]; + Certificate[] certificateChain = entry.certificateChain.toArray(new Certificate[entry.certificateChain.size()]); + keystore.setKeyEntry(entry.alias, entry.privateKey, password, certificateChain); + } + for (CertificateEntry entry : certificateEntries) { + keystore.setCertificateEntry(entry.alias, entry.certificate); + } + return keystore; + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static class KeyEntry { + final String alias; + final PrivateKey privateKey; + final List<X509Certificate> certificateChain; + final char[] password; + + KeyEntry(String alias, PrivateKey privateKey, List<X509Certificate> certificateChain, char[] password) { + this.alias = alias; + this.privateKey = privateKey; + this.certificateChain = certificateChain; + this.password = password; + } + } + + private static class CertificateEntry { + final String alias; + final X509Certificate certificate; + + CertificateEntry(String alias, X509Certificate certificate) { + this.alias = alias; + this.certificate = certificate; + } + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreType.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreType.java new file mode 100644 index 00000000000..a5c35549540 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreType.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.tls; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.KeyStoreException; + +/** + * @author bjorncs + */ +public enum KeyStoreType { + JKS { + KeyStore createKeystore() throws KeyStoreException { + return KeyStore.getInstance("JKS"); + } + }, + PKCS12 { + private final BouncyCastleProvider bouncyCastleProvider = new BouncyCastleProvider(); + + KeyStore createKeystore() throws KeyStoreException { + return KeyStore.getInstance("PKCS12", bouncyCastleProvider); + } + }; + abstract KeyStore createKeystore() throws GeneralSecurityException; +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreUtils.java new file mode 100644 index 00000000000..12aaa40cce4 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyStoreUtils.java @@ -0,0 +1,34 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.tls; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; + +/** + * @author bjorncs + */ +public class KeyStoreUtils { + private KeyStoreUtils() {} + + public static void writeKeyStoreToFile(KeyStore keyStore, File file, char[] password) { + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file))) { + keyStore.store(out, password); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + + } + + public static void writeKeyStoreToFile(KeyStore keyStore, File file) { + writeKeyStoreToFile(keyStore, file, new char[0]); + } + +} diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilderTest.java index 38f3b12b16b..20ac8791863 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilderTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/AthenzSslContextBuilderTest.java @@ -1,27 +1,19 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.tls; -import com.yahoo.athenz.auth.util.Crypto; -import com.yahoo.vespa.athenz.tls.AthenzSslContextBuilder.KeyStoreType; -import org.bouncycastle.asn1.x500.X500Name; -import org.bouncycastle.operator.OperatorCreationException; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.KeyStore; -import java.security.NoSuchAlgorithmException; -import java.security.cert.Certificate; import java.security.cert.X509Certificate; +import static com.yahoo.vespa.athenz.tls.TestUtils.createCertificate; +import static com.yahoo.vespa.athenz.tls.TestUtils.createKeyPair; +import static com.yahoo.vespa.athenz.tls.TestUtils.createKeystore; +import static com.yahoo.vespa.athenz.tls.TestUtils.createKeystoreFile; + /** * @author bjorncs */ @@ -35,22 +27,22 @@ public class AthenzSslContextBuilderTest { @Test public void can_build_sslcontext_with_truststore_only() throws Exception { new AthenzSslContextBuilder() - .withTrustStore(createKeystore("JKS")) + .withTrustStore(createKeystore(KeyStoreType.JKS, PASSWORD)) .build(); } @Test public void can_build_sslcontext_with_keystore_only() throws Exception { new AthenzSslContextBuilder() - .withKeyStore(createKeystore("JKS"), PASSWORD) + .withKeyStore(createKeystore(KeyStoreType.JKS, PASSWORD), PASSWORD) .build(); } @Test public void can_build_sslcontext_with_truststore_and_keystore() throws Exception { new AthenzSslContextBuilder() - .withKeyStore(createKeystore("JKS"), PASSWORD) - .withTrustStore(createKeystore("JKS")) + .withKeyStore(createKeystore(KeyStoreType.JKS, PASSWORD), PASSWORD) + .withTrustStore(createKeystore(KeyStoreType.JKS, PASSWORD)) .build(); } @@ -65,11 +57,9 @@ public class AthenzSslContextBuilderTest { @Test public void can_build_sslcontext_with_jks_keystore_from_file() throws Exception { - KeyStore keystore = createKeystore("JKS"); File keystoreFile = tempDirectory.newFile(); - try (OutputStream out = new BufferedOutputStream(new FileOutputStream(keystoreFile))) { - keystore.store(out, PASSWORD); - } + createKeystoreFile(keystoreFile, KeyStoreType.JKS, PASSWORD); + new AthenzSslContextBuilder() .withKeyStore(keystoreFile, PASSWORD, KeyStoreType.JKS) .build(); @@ -77,36 +67,12 @@ public class AthenzSslContextBuilderTest { @Test public void can_build_sslcontext_with_pcks12_keystore_from_file() throws Exception { - KeyStore keystore = createKeystore("PKCS12"); File keystoreFile = tempDirectory.newFile(); - try (OutputStream out = new BufferedOutputStream(new FileOutputStream(keystoreFile))) { - keystore.store(out, PASSWORD); - } + createKeystoreFile(keystoreFile, KeyStoreType.PKCS12, PASSWORD); + new AthenzSslContextBuilder() .withKeyStore(keystoreFile, PASSWORD, KeyStoreType.PKCS12) .build(); } - private static KeyStore createKeystore(String type) throws Exception { - KeyPair keyPair = createKeyPair(); - KeyStore keystore = KeyStore.getInstance(type); - keystore.load(null); - keystore.setKeyEntry("entry-name", keyPair.getPrivate(), PASSWORD, new Certificate[]{createCertificate(keyPair)}); - return keystore; - } - - private static X509Certificate createCertificate(KeyPair keyPair) throws - OperatorCreationException, IOException { - String x500Principal = "CN=mysubject"; - PKCS10CertificationRequest csr = - Crypto.getPKCS10CertRequest( - Crypto.generateX509CSR(keyPair.getPrivate(), x500Principal, null)); - return Crypto.generateX509Certificate(csr, keyPair.getPrivate(), new X500Name(x500Principal), 3600, false); - } - - private static KeyPair createKeyPair() throws NoSuchAlgorithmException { - KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); - keyGen.initialize(512); - return keyGen.genKeyPair(); - } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilderTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilderTest.java new file mode 100644 index 00000000000..1b6fa8bcbf1 --- /dev/null +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyStoreBuilderTest.java @@ -0,0 +1,54 @@ +package com.yahoo.vespa.athenz.tls; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.security.KeyPair; +import java.security.cert.X509Certificate; + +import static com.yahoo.vespa.athenz.tls.TestUtils.createCertificate; +import static com.yahoo.vespa.athenz.tls.TestUtils.createKeyPair; +import static com.yahoo.vespa.athenz.tls.TestUtils.createKeystoreFile; + +/** + * @author bjorncs + */ +public class KeyStoreBuilderTest { + + private static final char[] PASSWORD = new char[0]; + + @Rule + public TemporaryFolder tempDirectory = new TemporaryFolder(); + + @Test + public void can_create_jks_keystore_from_privatekey_and_certificate() throws Exception { + KeyPair keyPair = createKeyPair(); + X509Certificate certificate = createCertificate(keyPair); + KeyStoreBuilder.withType(KeyStoreType.JKS) + .withKeyEntry("key", keyPair.getPrivate(), certificate) + .build(); + } + + @Test + public void can_build_jks_keystore_from_file() throws Exception { + File keystoreFile = tempDirectory.newFile(); + createKeystoreFile(keystoreFile, KeyStoreType.JKS, PASSWORD); + + KeyStoreBuilder.withType(KeyStoreType.JKS) + .fromFile(keystoreFile, PASSWORD) + .build(); + } + + @Test + public void can_build_pcks12_keystore_from_file() throws Exception { + File keystoreFile = tempDirectory.newFile(); + createKeystoreFile(keystoreFile, KeyStoreType.PKCS12, PASSWORD); + + KeyStoreBuilder.withType(KeyStoreType.PKCS12) + .fromFile(keystoreFile, PASSWORD) + .build(); + } + +}
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java new file mode 100644 index 00000000000..54601c04514 --- /dev/null +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/TestUtils.java @@ -0,0 +1,57 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.tls; + +import com.yahoo.athenz.auth.util.Crypto; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.GeneralSecurityException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; + +import static com.yahoo.vespa.athenz.tls.KeyStoreUtils.writeKeyStoreToFile; + +/** + * @author bjorncs + */ +class TestUtils { + + static KeyStore createKeystore(KeyStoreType type, char[] password) + throws GeneralSecurityException, IOException, OperatorCreationException { + KeyPair keyPair = createKeyPair(); + KeyStore keystore = type.createKeystore(); + keystore.load(null); + keystore.setKeyEntry("entry-name", keyPair.getPrivate(), password, new Certificate[]{createCertificate(keyPair)}); + return keystore; + } + + static X509Certificate createCertificate(KeyPair keyPair) + throws OperatorCreationException, IOException { + String x500Principal = "CN=mysubject"; + PKCS10CertificationRequest csr = + Crypto.getPKCS10CertRequest( + Crypto.generateX509CSR(keyPair.getPrivate(), x500Principal, null)); + return Crypto.generateX509Certificate(csr, keyPair.getPrivate(), new X500Name(x500Principal), 3600, false); + } + + static KeyPair createKeyPair() throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(4096); + return keyGen.genKeyPair(); + } + + static void createKeystoreFile(File file, KeyStoreType type, char[] password) + throws IOException, GeneralSecurityException, OperatorCreationException { + writeKeyStoreToFile(createKeystore(type, password), file, password); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java index 01bf082d32f..17b62bf1a77 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; - import java.util.Arrays; /** @@ -10,7 +8,6 @@ import java.util.Arrays; * * @author bratseth */ -@Beta public final class DimensionSizes { private final long[] sizes; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 7130c053e9f..38979411313 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -20,7 +19,6 @@ import java.util.Set; * * @author bratseth */ -@Beta public class IndexedTensor implements Tensor { /** The prescribed and possibly abstract type this is an instance of */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index 15993072c37..ef19ef2e96c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableMap; import java.util.Iterator; @@ -12,7 +11,6 @@ import java.util.Map; * * @author bratseth */ -@Beta public class MappedTensor implements Tensor { private final TensorType type; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 0c9ed769c0d..5ff33aa340b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -2,7 +2,6 @@ package com.yahoo.tensor; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -23,7 +22,6 @@ import java.util.stream.Collectors; * * @author lesters */ -@Beta public class MixedTensor implements Tensor { /** The dimension specification for this tensor */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java index 23ef0772aea..9c41d5aad68 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/PartialAddress.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; - /** * An address to a subset of a tensors' cells, specifying a label for some but not necessarily all of the tensors * dimensions. @@ -13,7 +11,6 @@ import com.google.common.annotations.Beta; // - These are created in inner (though not inner-most) loops so they are implemented with minimal allocation. // We also avoid non-essential error checking. // - We can add support for string labels later without breaking the API -@Beta public class PartialAddress { // Two arrays which contains corresponding dimension=label pairs. diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 2255cf82338..59d5ee72372 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; import com.yahoo.tensor.functions.Argmax; import com.yahoo.tensor.functions.Argmin; import com.yahoo.tensor.functions.Concat; @@ -48,7 +47,6 @@ import java.util.function.Function; * * @author bratseth */ -@Beta public interface Tensor { // ----------------- Accessors diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 38553497478..1159d2fb32e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; - import java.util.Arrays; import java.util.Objects; import java.util.Optional; @@ -13,7 +11,6 @@ import java.util.Optional; * * @author bratseth */ -@Beta public abstract class TensorAddress implements Comparable<TensorAddress> { public static TensorAddress of(String[] labels) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java index 9b3a9328f07..a1334dc729c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java @@ -1,14 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; - import java.util.Optional; /** * @author bratseth */ -@Beta class TensorParser { static Tensor tensorFrom(String tensorString, Optional<TensorType> type) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 0176dac6821..c846364df29 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -23,7 +22,6 @@ import java.util.stream.Collectors; * @author geirst * @author bratseth */ -@Beta public class TensorType { /** The empty tensor type - which is the same as a double */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java index e3a194a96d7..32ad6171e57 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor; -import com.google.common.annotations.Beta; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -14,7 +12,6 @@ import java.util.regex.Pattern; * * @author geirst */ -@Beta public class TensorTypeParser { private final static String START_STRING = "tensor("; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/EvaluationContext.java b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/EvaluationContext.java index 8a969180113..9ec105f8174 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/EvaluationContext.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/EvaluationContext.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.evaluation; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; /** @@ -9,7 +8,6 @@ import com.yahoo.tensor.Tensor; * * @author bratseth */ -@Beta public interface EvaluationContext<NAMETYPE extends TypeContext.Name> extends TypeContext<NAMETYPE> { /** Returns the tensor bound to this name, or null if none */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/MapEvaluationContext.java b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/MapEvaluationContext.java index b9394da31e3..e302e317418 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/MapEvaluationContext.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/MapEvaluationContext.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.evaluation; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -10,7 +9,6 @@ import java.util.HashMap; /** * @author bratseth */ -@Beta public class MapEvaluationContext implements EvaluationContext<TypeContext.Name> { private final java.util.Map<String, Tensor> bindings = new HashMap<>(); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java index acb2363cba4..c1cfa319664 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.evaluation; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.PrimitiveTensorFunction; @@ -17,7 +16,6 @@ import java.util.Optional; * * @author bratseth */ -@Beta public class VariableTensor extends PrimitiveTensorFunction { private final String name; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java index 93365d20966..3478061b32c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; - import java.util.Collections; import java.util.List; /** * @author bratseth */ -@Beta public class Argmax extends CompositeTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java index e598cdf8a98..ba5b3c3e4b2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; - import java.util.Collections; import java.util.List; /** * @author bratseth */ -@Beta public class Argmin extends CompositeTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/CompositeTensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/CompositeTensorFunction.java index bfc0938abcc..2a52df20108 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/CompositeTensorFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/CompositeTensorFunction.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.EvaluationContext; @@ -13,7 +12,6 @@ import com.yahoo.tensor.evaluation.TypeContext; * * @author bratseth */ -@Beta public abstract class CompositeTensorFunction extends TensorFunction { /** Finds the type this produces by first converting it to a primitive function */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java index 13e7c136feb..91ab4f9d046 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java @@ -1,10 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; -import com.yahoo.lang.MutableInteger; -import com.yahoo.lang.MutableLong; import com.yahoo.tensor.DimensionSizes; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; @@ -26,7 +23,6 @@ import java.util.stream.Collectors; * * @author bratseth */ -@Beta public class Concat extends PrimitiveTensorFunction { private final TensorFunction argumentA, argumentB; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java index a43de297b9a..7c1ce068c90 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.EvaluationContext; @@ -15,7 +14,6 @@ import java.util.List; * * @author bratseth */ -@Beta public class ConstantTensor extends PrimitiveTensorFunction { private final Tensor constant; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java index edfa8253eb9..83cba3479e2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.yahoo.tensor.DimensionSizes; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; @@ -19,7 +18,6 @@ import java.util.function.Function; * * @author bratseth */ -@Beta public class Generate extends PrimitiveTensorFunction { private final TensorType type; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 54568401adb..be323313369 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.yahoo.tensor.DimensionSizes; @@ -32,7 +31,6 @@ import java.util.function.DoubleBinaryOperator; * * @author bratseth */ -@Beta public class Join extends PrimitiveTensorFunction { private final TensorFunction argumentA, argumentB; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java index d7f7ae59d62..7939457a101 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; - import java.util.Collections; import java.util.List; /** * @author bratseth */ -@Beta public class L1Normalize extends CompositeTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java index e2c526760bd..40edb8ba23f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java @@ -1,15 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; - import java.util.Collections; import java.util.List; /** * @author bratseth */ -@Beta public class L2Normalize extends CompositeTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java index 4a338e5501e..016c60c6897 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; @@ -19,7 +18,6 @@ import java.util.function.DoubleUnaryOperator; * * @author bratseth */ -@Beta public class Map extends PrimitiveTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java index 935e4761cfe..7c65afc98f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.yahoo.tensor.TensorType; @@ -10,7 +9,6 @@ import java.util.List; /** * @author bratseth */ -@Beta public class Matmul extends CompositeTensorFunction { private final TensorFunction argument1, argument2; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/PrimitiveTensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/PrimitiveTensorFunction.java index b7c9a5d2342..e2aae39f11f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/PrimitiveTensorFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/PrimitiveTensorFunction.java @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; -import com.yahoo.tensor.Tensor; - /** * A primitive tensor function is a tensor function which cannot be expressed in terms of other tensor functions. * All tensor implementations must implement all primitive tensor functions. @@ -11,7 +8,6 @@ import com.yahoo.tensor.Tensor; * * @author bratseth */ -@Beta public abstract class PrimitiveTensorFunction extends TensorFunction { } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java index 1475f7f4ac1..7175c91ed33 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java @@ -2,12 +2,6 @@ package com.yahoo.tensor.functions; import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.functions.CompositeTensorFunction; -import com.yahoo.tensor.functions.Generate; -import com.yahoo.tensor.functions.PrimitiveTensorFunction; -import com.yahoo.tensor.functions.ScalarFunctions; -import com.yahoo.tensor.functions.TensorFunction; -import com.yahoo.tensor.functions.ToStringContext; import java.util.Collections; import java.util.List; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java index e045effbe7e..332150f957d 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; @@ -25,7 +24,6 @@ import java.util.Set; * * @author bratseth */ -@Beta public class Reduce extends PrimitiveTensorFunction { public enum Aggregator { avg, count, prod, sum, max, min; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java index af4492ca1e4..53d774de329 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; @@ -21,7 +20,6 @@ import java.util.Objects; * * @author bratseth */ -@Beta public class Rename extends PrimitiveTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java index 92c1c0307ec..944755c9db2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import java.util.List; @@ -18,7 +17,6 @@ import java.util.stream.Collectors; * * @author bratseth */ -@Beta public class ScalarFunctions { public static DoubleBinaryOperator add() { return new Add(); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java index 32cff5ac84a..bd732cdc11e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import com.yahoo.tensor.TensorType; @@ -11,7 +10,6 @@ import java.util.List; /** * @author bratseth */ -@Beta public class Softmax extends CompositeTensorFunction { private final TensorFunction argument; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java index e805e9d87bb..1086d91da31 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.EvaluationContext; @@ -17,7 +16,6 @@ import java.util.List; * * @author bratseth */ -@Beta public abstract class TensorFunction { /** Returns the function arguments of this node in the order they are applied */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java index 62293a7e0ac..cb7f376c365 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ToStringContext.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; - /** * A context which is passed down to all nested functions when returning a string representation. * The default implementation is empty as this library does not in itself have any need for a @@ -10,7 +8,6 @@ import com.google.common.annotations.Beta; * * @author bratseth */ -@Beta public interface ToStringContext { static ToStringContext empty() { return new ToStringContext() {}; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java index 78ff0731566..4c0748ee39a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; -import com.google.common.annotations.Beta; import com.google.common.collect.ImmutableList; import java.util.List; @@ -9,7 +8,6 @@ import java.util.List; /** * @author bratseth */ -@Beta public class XwPlusB extends CompositeTensorFunction { private final TensorFunction x, w, b; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java index 416b28afa22..4ae25596a2b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.serialization; -import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -14,7 +13,6 @@ import java.util.Optional; * * @author geirst */ -@Beta interface BinaryFormat { /** diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java index 1e830bac461..46d533cc6bd 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.serialization; -import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.DimensionSizes; import com.yahoo.tensor.IndexedTensor; @@ -21,7 +20,6 @@ import java.util.Optional; * * @author bratseth */ -@Beta public class DenseBinaryFormat implements BinaryFormat { @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java index 34e6cccf0f0..acaeb3ef5ba 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java @@ -2,7 +2,6 @@ package com.yahoo.tensor.serialization; -import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.Tensor; @@ -20,7 +19,6 @@ import java.util.stream.Collectors; * * @author lesters */ -@Beta class MixedBinaryFormat implements BinaryFormat { @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java index 0cd3ff77aca..190b31b7b35 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.serialization; -import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; @@ -23,7 +22,6 @@ import java.util.Optional; * * @author geirst */ -@Beta class SparseBinaryFormat implements BinaryFormat { @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java index 01a1d023f2b..1c18adc8652 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.serialization; -import com.google.common.annotations.Beta; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.MixedTensor; @@ -19,7 +18,6 @@ import java.util.Optional; * * @author geirst */ -@Beta public class TypedBinaryFormat { private static final int SPARSE_BINARY_FORMAT_TYPE = 1; |