diff options
67 files changed, 1408 insertions, 396 deletions
diff --git a/client/README.md b/client/README.md index ddea3591e38..d08b19a1a12 100644 --- a/client/README.md +++ b/client/README.md @@ -1,12 +1,12 @@ # vespa_query_dsl This lib is used for composing vespa YQL queries -referece: https://docs.vespa.ai/documentation/reference/query-language-reference.html +Reference: https://docs.vespa.ai/documentation/reference/query-language-reference.html # usage -please refer the unit test: +Please refer to the unit test: -https://github.com/vespa-engine/vespa/blob/master/client/src/test/groovy/com/yahoo/vespa/client/dsl/QTest.groovy +https://github.com/vespa-engine/vespa/tree/master/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy # todos - [ ] support `predicate` (https://docs.vespa.ai/documentation/predicate-fields.html) diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 6b466c65cdb..0f5a5e6271d 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -605,5 +605,511 @@ "public static final com.yahoo.config.application.api.ValidationOverrides empty", "public static final com.yahoo.config.application.api.ValidationOverrides all" ] + }, + "com.yahoo.config.model.api.ApplicationInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.config.provision.ApplicationId, long, com.yahoo.config.model.api.Model)", + "public com.yahoo.config.provision.ApplicationId getApplicationId()", + "public long getGeneration()", + "public com.yahoo.config.model.api.Model getModel()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigChangeAction$Type": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ConfigChangeAction$Type[] values()", + "public static com.yahoo.config.model.api.ConfigChangeAction$Type valueOf(java.lang.String)", + "public java.lang.String toString()" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type RESTART", + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type REFEED" + ] + }, + "com.yahoo.config.model.api.ConfigChangeAction": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public abstract java.lang.String getMessage()", + "public abstract java.util.List getServices()", + "public abstract boolean allowed()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigChangeRefeedAction": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public java.lang.String name()", + "public abstract java.lang.String getDocumentType()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigChangeRestartAction": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public boolean allowed()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigDefinitionRepo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.util.Map getConfigDefinitions()", + "public abstract com.yahoo.vespa.config.buildergen.ConfigDefinition get(com.yahoo.vespa.config.ConfigDefinitionKey)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigDefinitionStore": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.vespa.config.ConfigDefinition getConfigDefinition(com.yahoo.vespa.config.ConfigDefinitionKey)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigModelPlugin": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigServerSpec": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.lang.String getHostName()", + "public abstract int getConfigServerPort()", + "public int getHttpPort()", + "public abstract int getZooKeeperPort()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ContainerEndpoint": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.util.List)", + "public java.lang.String clusterId()", + "public java.util.List names()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.EndpointCertificateMetadata": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String, int)", + "public java.lang.String keyName()", + "public java.lang.String certName()", + "public int version()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.EndpointCertificateSecrets": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String)", + "public java.lang.String certificate()", + "public java.lang.String key()", + "public boolean isMissing()" + ], + "fields": [ + "public static final com.yahoo.config.model.api.EndpointCertificateSecrets MISSING" + ] + }, + "com.yahoo.config.model.api.FileDistribution": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void startDownload(java.lang.String, int, java.util.Set)", + "public abstract java.io.File getFileReferencesDir()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.HostInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.util.Collection)", + "public java.lang.String getHostname()", + "public java.util.Collection getServices()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.HostProvisioner": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.provision.HostSpec allocateHost(java.lang.String)", + "public abstract java.util.List prepare(com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, int, com.yahoo.config.provision.ProvisionLogger)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.Model": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.vespa.config.ConfigPayload getConfig(com.yahoo.vespa.config.ConfigKey, com.yahoo.vespa.config.buildergen.ConfigDefinition)", + "public abstract java.util.Set allConfigsProduced()", + "public abstract java.util.Collection getHosts()", + "public abstract java.util.Set allConfigIds()", + "public abstract void distributeFiles(com.yahoo.config.model.api.FileDistribution)", + "public abstract java.util.Set fileReferences()", + "public abstract com.yahoo.config.provision.AllocatedHosts allocatedHosts()", + "public boolean allowModelVersionMismatch(java.time.Instant)", + "public boolean skipOldConfigModels(java.time.Instant)", + "public com.yahoo.component.Version version()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelContext$Properties": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract boolean multitenant()", + "public abstract com.yahoo.config.provision.ApplicationId applicationId()", + "public abstract java.util.List configServerSpecs()", + "public abstract com.yahoo.config.provision.HostName loadBalancerName()", + "public abstract java.net.URI ztsUrl()", + "public abstract java.lang.String athenzDnsSuffix()", + "public abstract boolean hostedVespa()", + "public abstract com.yahoo.config.provision.Zone zone()", + "public abstract java.util.Set endpoints()", + "public abstract boolean isBootstrap()", + "public abstract boolean isFirstTimeDeployment()", + "public boolean useDedicatedNodeForLogserver()", + "public abstract boolean useAdaptiveDispatch()", + "public java.util.Optional tlsSecrets()", + "public java.util.Optional endpointCertificateSecrets()", + "public abstract double defaultTermwiseLimit()", + "public abstract boolean useBucketSpaceMetric()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelContext": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.application.api.ApplicationPackage applicationPackage()", + "public abstract java.util.Optional previousModel()", + "public abstract java.util.Optional permanentApplicationPackage()", + "public abstract java.util.Optional hostProvisioner()", + "public abstract com.yahoo.config.application.api.DeployLogger deployLogger()", + "public abstract com.yahoo.config.model.api.ConfigDefinitionRepo configDefinitionRepo()", + "public abstract com.yahoo.config.application.api.FileRegistry getFileRegistry()", + "public abstract com.yahoo.config.model.api.ModelContext$Properties properties()", + "public java.util.Optional appDir()", + "public abstract com.yahoo.component.Version modelVespaVersion()", + "public abstract com.yahoo.component.Version wantedNodeVespaVersion()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelCreateResult": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.config.model.api.Model, java.util.List)", + "public com.yahoo.config.model.api.Model getModel()", + "public java.util.List getConfigChangeActions()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelFactory": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.component.Version version()", + "public abstract com.yahoo.config.model.api.Model createModel(com.yahoo.config.model.api.ModelContext)", + "public abstract com.yahoo.config.model.api.ModelCreateResult createAndValidateModel(com.yahoo.config.model.api.ModelContext, com.yahoo.config.model.api.ValidationParameters)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelState": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.model.api.Model getModel()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.PortInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(int, java.util.Collection)", + "public int getPort()", + "public java.util.Collection getTags()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ServiceInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String, java.util.Collection, java.util.Map, java.lang.String, java.lang.String)", + "public java.lang.String getServiceName()", + "public java.lang.String getConfigId()", + "public java.lang.String getServiceType()", + "public java.util.Optional getProperty(java.lang.String)", + "public java.util.Collection getPorts()", + "public java.lang.String getHostName()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.SuperModel": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(java.util.Map)", + "public java.util.Map getModelsPerTenant()", + "public java.util.Map getModels()", + "public java.util.List getAllApplicationInfos()", + "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)", + "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)", + "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.SuperModelListener": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)", + "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.SuperModelProvider": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void registerListener(com.yahoo.config.model.api.SuperModelListener)", + "public abstract com.yahoo.config.model.api.SuperModel getSuperModel()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.TlsSecrets": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String)", + "public void <init>(com.yahoo.config.model.api.EndpointCertificateSecrets)", + "public java.lang.String certificate()", + "public java.lang.String key()", + "public boolean isMissing()" + ], + "fields": [ + "public static final com.yahoo.config.model.api.TlsSecrets MISSING" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$CheckRouting": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors)", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$CheckRouting)", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors, com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange, com.yahoo.config.model.api.ValidationParameters$CheckRouting)", + "public boolean ignoreValidationErrors()", + "public boolean failOnIncompatibleChanges()", + "public boolean checkRouting()" + ], + "fields": [] } }
\ No newline at end of file diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java new file mode 100644 index 00000000000..a1fae9bb148 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java @@ -0,0 +1,35 @@ +package com.yahoo.config.model.api; + +public class EndpointCertificateMetadata { + + private final String keyName; + private final String certName; + private final int version; + + public EndpointCertificateMetadata(String keyName, String certName, int version) { + this.keyName = keyName; + this.certName = certName; + this.version = version; + } + + public String keyName() { + return keyName; + } + + public String certName() { + return certName; + } + + public int version() { + return version; + } + + @Override + public String toString() { + return "EndpointCertificateMetadata{" + + "keyName='" + keyName + '\'' + + ", certName='" + certName + '\'' + + ", version=" + version + + '}'; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java new file mode 100644 index 00000000000..6fcbac4f422 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java @@ -0,0 +1,30 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +public class EndpointCertificateSecrets { + public static final EndpointCertificateSecrets MISSING = new EndpointCertificateSecrets(); + + private final String certificate; + private final String key; + + private EndpointCertificateSecrets() { + this(null, null); + } + + public EndpointCertificateSecrets(String certificate, String key) { + this.certificate = certificate; + this.key = key; + } + + public String certificate() { + return certificate; + } + + public String key() { + return key; + } + + public boolean isMissing() { + return this == MISSING; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 323aa473580..81ac02a5400 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -54,8 +54,9 @@ public interface ModelContext { // TODO: Remove when Vespa 7.112 is the oldest config model in use default boolean useDedicatedNodeForLogserver() { return true; } boolean useAdaptiveDispatch(); - // TODO: Remove temporary default implementation + // TODO: Remove temporary default implementations default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } + default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } double defaultTermwiseLimit(); boolean useBucketSpaceMetric(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java index 6a8b5a237ab..0937b8b77ec 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java @@ -16,6 +16,11 @@ public class TlsSecrets { this.key = key; } + public TlsSecrets(EndpointCertificateSecrets endpointCertificateSecrets) { + this.certificate = endpointCertificateSecrets.certificate(); + this.key = endpointCertificateSecrets.key(); + } + public String certificate() { return certificate; } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index 52ce35a19fb..a3478026520 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -1,5 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi // Not really "public", only annotated as such to enable the ABI checker plugin package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.api.annotations.PublicApi; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index b286b94c699..7c9e930bb4f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -15,7 +15,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -255,7 +255,7 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } - public Optional<TlsSecrets> tlsSecrets() { return properties.tlsSecrets(); } + public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); } public Optional<String> tlsClientAuthority() { var caFile = applicationPackage.getClientSecurityFile(); @@ -289,7 +289,6 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; - private Optional<TlsSecrets> tlsSecrets = Optional.empty(); public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9d561a79c75..9f4d1b09f91 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -39,7 +40,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useDedicatedNodeForLogserver = false; private boolean useAdaptiveDispatch = false; private double defaultTermwiseLimit = 1.0; - private Optional<TlsSecrets> tlsSecrets = Optional.empty(); + private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); @Override public boolean multitenant() { return multitenant; } @@ -56,7 +57,8 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } - @Override public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } + @Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } @@ -95,9 +97,8 @@ public class TestProperties implements ModelContext.Properties { return this; } - - public TestProperties setTlsSecrets(Optional<TlsSecrets> tlsSecrets) { - this.tlsSecrets = tlsSecrets; + public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { + this.endpointCertificateSecrets = endpointCertificateSecrets; return this; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 58ef47b7ba9..441d1b5b8df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -69,7 +69,7 @@ public class RankProfileTypeSettingsProcessor extends Processor { } private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) { - for (RankProfile profile : rankProfileRegistry.all()) { + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { profile.addAttributeType(attributeName, attributeType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index 2f972b8ecb3..f00ad0f0dbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -1,17 +1,17 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.vespa.model.VespaModel; -public class TlsSecretsValidator extends Validator { +public class EndpointCertificateSecretsValidator extends Validator { /** This check is delayed until validation to allow node provisioning to complete while we are waiting for cert */ @Override public void validate(VespaModel model, DeployState deployState) { - if (deployState.tlsSecrets().isPresent() && deployState.tlsSecrets().get() == TlsSecrets.MISSING) { + if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 8eabc61f71f..1e4a45428b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -57,7 +57,7 @@ public class Validation { new DeploymentSpecValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 5e0dde6161d..efd00528d54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -5,7 +5,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; @@ -55,7 +54,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private ContainerModelEvaluation modelEvaluation; - private Optional<TlsSecrets> tlsSecrets; private Optional<String> tlsClientAuthority; private MbusParams mbusParams; @@ -65,8 +63,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState); - - this.tlsSecrets = deployState.tlsSecrets(); this.tlsClientAuthority = deployState.tlsClientAuthority(); restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -205,10 +201,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } - public Optional<TlsSecrets> getTlsSecrets() { - return tlsSecrets; - } - public Optional<String> getTlsClientAuthority() { return tlsClientAuthority; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 7a08a3c1a7b..12db3b87243 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -23,15 +23,15 @@ public class HostedSslConnectorFactory extends ConnectorFactory { /** * Create connector factory that uses a certificate provided by the config-model / configserver. */ - public static HostedSslConnectorFactory withProvidedCertificate(String serverName, TlsSecrets tlsSecrets) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, /*tlsCaCertificates*/null), false); + public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificates*/null), false); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ - public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, tlsCaCertificates), true); + public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, tlsCaCertificates), true); } /** @@ -47,11 +47,11 @@ public class HostedSslConnectorFactory extends ConnectorFactory { } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( - String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { return new ConfiguredDirectSslProvider( serverName, - tlsSecrets.key(), - tlsSecrets.certificate(), + endpointCertificateSecrets.key(), + endpointCertificateSecrets.certificate(), /*caCertificatePath*/null, tlsCaCertificates, ClientAuth.Enum.WANT_AUTH); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3da0b01f614..aef2697a5dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -13,7 +13,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; @@ -327,15 +327,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port - if (deployState.tlsSecrets().isPresent()) { + if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); } - TlsSecrets tlsSecrets = deployState.tlsSecrets().get(); + EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, tlsSecrets, deployState.tlsClientAuthority().get()) - : HostedSslConnectorFactory.withProvidedCertificate(serverName, tlsSecrets); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets); server.addConnector(connectorFactory); } else { server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); diff --git a/config-model/src/test/derived/tensor2/first.sd b/config-model/src/test/derived/tensor2/first.sd new file mode 100644 index 00000000000..80554572503 --- /dev/null +++ b/config-model/src/test/derived/tensor2/first.sd @@ -0,0 +1,8 @@ +search first { + document first { + field first_field type tensor(first[10]) { + indexing: summary | attribute + } + } +} + diff --git a/config-model/src/test/derived/tensor2/rank-profiles.cfg b/config-model/src/test/derived/tensor2/rank-profiles.cfg new file mode 100644 index 00000000000..7e087832042 --- /dev/null +++ b/config-model/src/test/derived/tensor2/rank-profiles.cfg @@ -0,0 +1,14 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "vespa.type.attribute.second_field" +rankprofile[].fef.property[].value "tensor(second[10])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.second_field" +rankprofile[].fef.property[].value "tensor(second[10])" diff --git a/config-model/src/test/derived/tensor2/second.sd b/config-model/src/test/derived/tensor2/second.sd new file mode 100644 index 00000000000..ace0540c8bd --- /dev/null +++ b/config-model/src/test/derived/tensor2/second.sd @@ -0,0 +1,7 @@ +search second { + document second { + field second_field type tensor(second[10]) { + indexing: summary | attribute + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index ebd2c752d5e..61065cd4bcc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -138,4 +139,15 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("tensor"); } + @Test + public void testTensor2() throws IOException, ParseException { + String dir = "src/test/derived/tensor2/"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(dir + "first.sd"); + builder.importFile(dir + "second.sd"); + builder.build(); + derive("tensor2", builder, builder.getSearch("second")); + assertCorrectConfigFiles("tensor2"); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index cdb4ce955e2..21df39ebde8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; /** * @author andreer */ -public class TlsSecretsValidatorTest { +public class EndpointCertificateSecretsValidatorTest { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @@ -43,21 +43,21 @@ public class TlsSecretsValidatorTest { @Test public void missing_certificate_fails_validation() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(TlsSecrets.MISSING)); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(EndpointCertificateSecrets.MISSING)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } @Test public void validation_succeeds_with_certificate() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new TlsSecrets("cert", "key"))); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new EndpointCertificateSecrets("cert", "key"))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } @Test @@ -65,10 +65,10 @@ public class TlsSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.empty()); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } - private static DeployState deployState(String servicesXml, String deploymentXml, Optional<TlsSecrets> tlsSecrets) { + private static DeployState deployState(String servicesXml, String deploymentXml, Optional<EndpointCertificateSecrets> endpointCertificateSecretsSecrets) { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) .withDeploymentSpec(deploymentXml) @@ -79,7 +79,7 @@ public class TlsSecretsValidatorTest { .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(tlsSecrets)); + .setEndpointCertificateSecrets(endpointCertificateSecretsSecrets)); final DeployState deployState = builder.build(); assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 54d1c1c9793..1bbc4ea2684 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -5,7 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -693,7 +693,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) .zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName())) .build(); createModel(root, state, null, clusterElem); @@ -772,13 +772,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatProvidingTlsSecretOpensPort4443() { + public void requireThatProvidingEndpointCertificateSecretsOpensPort4443() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", nodesXml, "</container>" ); - DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build(); + DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build(); createModel(root, state, null, clusterElem); ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 863781073f8..68f507c810d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -258,7 +258,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) .modelHostProvisioner(new HostsXmlProvisioner(new StringReader(hostsxml))) .build(); MockRoot root = new MockRoot("root", deployState); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index d2f26738301..e9032555b09 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -26,6 +26,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; @@ -361,15 +362,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // until the config server where the deployment happened picks it up and deletes // the local session long sessionId = activeSession.get(); - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - - if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - } else { - throw new InternalServerException("Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + RemoteSession remoteSession; + try { + remoteSession = getRemoteSession(tenant, sessionId); + Transaction deleteTransaction = remoteSession.createDeleteTransaction(); + deleteTransaction.commit(); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + + if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + } else { + deleteTransaction.rollbackOrLog(); + throw new InternalServerException(applicationId + " was not deleted (waited " + waitTime + "), session " + sessionId); + } + } catch (NotFoundException e) { + // For the case where waiting timed out in a previous attempt at deleting the application, continue and do the steps below + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); } NestedTransaction transaction = new NestedTransaction(); @@ -468,6 +477,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 52d47a9398b..8b2c3e2cb0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -130,7 +131,7 @@ public class ModelContextImpl implements ModelContext { private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useAdaptiveDispatch; - private final Optional<TlsSecrets> tlsSecrets; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final boolean useBucketSpaceMetric; @@ -146,7 +147,7 @@ public class ModelContextImpl implements ModelContext { boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, - Optional<TlsSecrets> tlsSecrets) { + Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -160,7 +161,7 @@ public class ModelContextImpl implements ModelContext { this.isFirstTimeDeployment = isFirstTimeDeployment; this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.tlsSecrets = tlsSecrets; + this.endpointCertificateSecrets = endpointCertificateSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource) @@ -208,7 +209,10 @@ public class ModelContextImpl implements ModelContext { public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override - public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } + + @Override + public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index bc6419f230f..a2fc2bfd6a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -27,8 +27,9 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; @@ -135,7 +136,10 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating flagSource, - new TlsSecretsKeys(curator, TenantRepository.getTenantPath(tenant), secretStore).readTlsSecretsKeyFromZookeeper(applicationId)); + new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) + .readEndpointCertificateMetadata(applicationId) + .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets)); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index ab3e0e863ce..1a41c1efd7a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -11,6 +12,7 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import java.time.Clock; import java.time.Duration; @@ -32,6 +34,7 @@ public final class PrepareParams { static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; + static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -42,10 +45,12 @@ public final class PrepareParams { private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, - List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName) { + List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, + Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -55,6 +60,7 @@ public final class PrepareParams { this.vespaVersion = vespaVersion; this.containerEndpoints = containerEndpoints; this.tlsSecretsKeyName = tlsSecretsKeyName; + this.endpointCertificateMetadata = endpointCertificateMetadata; } public static class Builder { @@ -68,6 +74,7 @@ public final class PrepareParams { private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = List.of(); private Optional<String> tlsSecretsKeyName = Optional.empty(); + private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); public Builder() { } @@ -128,9 +135,16 @@ public final class PrepareParams { return this; } + public Builder endpointCertificateMetadata(String serialized) { + if(serialized == null) return this; + Slime slime = SlimeUtils.jsonToSlime(serialized); + endpointCertificateMetadata = Optional.of(EndpointCertificateMetadataSerializer.fromSlime(slime.get())); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName); + verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata); } } @@ -144,6 +158,7 @@ public final class PrepareParams { .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) + .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .build(); } @@ -200,4 +215,8 @@ public final class PrepareParams { public Optional<String> tlsSecretsKeyName() { return tlsSecretsKeyName; } + + public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { + return endpointCertificateMetadata; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 171eab35507..0115876ded9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -12,8 +12,9 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -33,7 +34,9 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import org.xml.sax.SAXException; @@ -113,7 +116,7 @@ public class SessionPreparer { preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { preparation.writeStateZK(); - preparation.writeTlsZK(); + preparation.writeEndpointCertificateMetadataZK(); preparation.writeContainerEndpointsZK(); preparation.distribute(); } @@ -142,8 +145,10 @@ public class SessionPreparer { final ContainerEndpointsCache containerEndpoints; final Set<ContainerEndpoint> endpointsSet; final ModelContext.Properties properties; - private final TlsSecretsKeys tlsSecretsKeys; - private final Optional<TlsSecrets> tlsSecrets; + private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; + private final EndpointCertificateRetriever endpointCertificateRetriever; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -162,8 +167,16 @@ public class SessionPreparer { this.applicationId = params.getApplicationId(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); - this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); - this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId); + this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); + this.endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); + + this.endpointCertificateMetadata = params.endpointCertificateMetadata() + .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); + + endpointCertificateSecrets = endpointCertificateMetadata + .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + this.endpointsSet = getEndpoints(params.containerEndpoints()); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), @@ -178,7 +191,7 @@ public class SessionPreparer { params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), - tlsSecrets); + endpointCertificateSecrets); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -233,9 +246,10 @@ public class SessionPreparer { checkTimeout("write state to zookeeper"); } - void writeTlsZK() { - tlsSecretsKeys.writeTlsSecretsKeyToZooKeeper(applicationId, params.tlsSecretsKeyName().orElse(null)); - checkTimeout("write tlsSecretsKey to zookeeper"); + void writeEndpointCertificateMetadataZK() { + endpointCertificateMetadata.ifPresent(metadata -> + endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, metadata)); + checkTimeout("write endpoint certificate metadata to zookeeper"); } void writeContainerEndpointsZK() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java new file mode 100644 index 00000000000..6d092aaa18b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +/** + * (de)serializes endpoint certificate metadata + * + * @author andreer + */ +public class EndpointCertificateMetadataSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private final static String keyNameField = "keyName"; + private final static String certNameField = "certName"; + private final static String versionField = "version"; + + public static void toSlime(EndpointCertificateMetadata metadata, Cursor object) { + object.setString(keyNameField, metadata.keyName()); + object.setString(certNameField, metadata.certName()); + object.setLong(versionField, metadata.version()); + } + + public static EndpointCertificateMetadata fromSlime(Inspector inspector) { + switch (inspector.type()) { + case STRING: // TODO: Remove once all are transmitted and stored as JSON + return new EndpointCertificateMetadata( + inspector.asString() + "-key", + inspector.asString() + "-cert", + 0 + ); + case OBJECT: + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()) + ); + + default: + throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); + } + } + + public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) { + return fromSlime(new Slime().setString(tlsSecretsKeys)); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java new file mode 100644 index 00000000000..6500449e557 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Optional; + +/** + * Stores the endpoint certificate metadata for an application. + * This metadata is then used to retrieve the actual secrets from {@link EndpointCertificateRetriever}. + * + * @author andreer + */ +public class EndpointCertificateMetadataStore { + + private final Path path; + private final Curator curator; + + public EndpointCertificateMetadataStore(Curator curator, Path tenantPath) { + this.curator = curator; + this.path = tenantPath.append("tlsSecretsKeys/"); + } + + /** Reads the endpoint certificate metadata from ZooKeeper, if it exists */ + public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId application) { + try { + Optional<byte[]> data = curator.getData(endpointCertificateMetadataPathOf(application)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + Slime slime = SlimeUtils.jsonToSlime(data.get()); + EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get()); + return Optional.of(endpointCertificateMetadata); + } catch (Exception e) { + throw new RuntimeException("Error reading TLS secret key of " + application, e); + } + } + + /** Writes the endpoint certificate metadata to ZooKeeper */ + public void writeEndpointCertificateMetadata(ApplicationId application, EndpointCertificateMetadata endpointCertificateMetadata) { + try { + Slime slime = new Slime(); + EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject()); + curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime)); + } catch (Exception e) { + throw new RuntimeException("Could not write TLS secret key of " + application, e); + } + } + + /** Returns a transaction which deletes these tls secrets key if they exist */ + public CuratorTransaction delete(ApplicationId application) { + if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator); + return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator); + } + + /** Returns the path storing the tls secrets key for an application */ + private Path endpointCertificateMetadataPathOf(ApplicationId application) { + return path.append(application.serializedForm()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java new file mode 100644 index 00000000000..5f40e5e1411 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; + +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.cert.X509Certificate; +import java.util.Optional; + +/** + * Used to retrieve actual endpoint certificate/key from secret store. + * + * @author andreer + */ +public class EndpointCertificateRetriever { + + private final SecretStore secretStore; + + public EndpointCertificateRetriever(SecretStore secretStore) { + this.secretStore = secretStore; + } + + public Optional<EndpointCertificateSecrets> readEndpointCertificateSecrets(EndpointCertificateMetadata metadata) { + return Optional.of(readFromSecretStore(metadata)); + } + + private EndpointCertificateSecrets readFromSecretStore(EndpointCertificateMetadata endpointCertificateMetadata) { + try { + String cert = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); + String key = secretStore.getSecret(endpointCertificateMetadata.keyName(), endpointCertificateMetadata.version()); + + verifyKeyMatchesCertificate(endpointCertificateMetadata, cert, key); + + return new EndpointCertificateSecrets(cert, key); + } catch (RuntimeException e) { + // Assume not ready yet + return EndpointCertificateSecrets.MISSING; + } + } + + private void verifyKeyMatchesCertificate(EndpointCertificateMetadata endpointCertificateMetadata, String cert, String key) { + X509Certificate x509Certificate = X509CertificateUtils.fromPem(cert); + + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(key); + PublicKey publicKey = x509Certificate.getPublicKey(); + + if(!X509CertificateUtils.privateKeyMatchesPublicKey(privateKey, publicKey)) { + throw new IllegalArgumentException("Failed to retrieve endpoint secrets: Certificate and key data do not match for " + endpointCertificateMetadata); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java deleted file mode 100644 index da6fc490da9..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.TlsSecrets; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.path.Path; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.transaction.CuratorOperations; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; - -import java.util.Optional; - -/** - * TLS Secret keys for applications (used to retrieve actual certificate/key from secret store). Persisted in ZooKeeper. - * - * @author andreer - */ -public class TlsSecretsKeys { - - private final Path path; - private final SecretStore secretStore; - private final Curator curator; - - public TlsSecretsKeys(Curator curator, Path tenantPath, SecretStore secretStore) { - this.curator = curator; - this.path = tenantPath.append("tlsSecretsKeys/"); - this.secretStore = secretStore; - } - - public Optional<TlsSecrets> readTlsSecretsKeyFromZookeeper(ApplicationId application) { - try { - Optional<byte[]> data = curator.getData(tlsSecretsKeyOf(application)); - if (data.isEmpty() || data.get().length == 0) return Optional.empty(); - - Slime slime = SlimeUtils.jsonToSlime(data.get()); - final var inspector = slime.get(); - - switch (inspector.type()) { - case STRING: // TODO: Remove once all are stored as JSON - return readFromSecretStore(Optional.ofNullable(inspector.asString())); - case OBJECT: - var tlsSecretsInfo = new TlsSecretsMetadata(); - tlsSecretsInfo.certName = inspector.field("certName").asString(); - tlsSecretsInfo.keyName = inspector.field("keyName").asString(); - tlsSecretsInfo.version = Math.toIntExact(inspector.field("version").asLong()); - return Optional.of(readFromSecretStore(tlsSecretsInfo)); - default: - throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); - } - } catch (Exception e) { - throw new RuntimeException("Error reading TLS secret key of " + application, e); - } - } - - public void writeTlsSecretsKeyToZooKeeper(ApplicationId application, String tlsSecretsKey) { - if (tlsSecretsKey == null) return; - writeTlsSecretsAsString(application, tlsSecretsKey); - } - - private void writeTlsSecretsAsString(ApplicationId application, String tlsSecretsKey) { - try { - Slime slime = new Slime(); - slime.setString(tlsSecretsKey); - curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - void writeTlsSecretsMetadata(ApplicationId application, TlsSecretsMetadata tlsSecretsMetadata) { - try { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString(TlsSecretsMetadata.certNameField, tlsSecretsMetadata.certName); - cursor.setString(TlsSecretsMetadata.keyNameField, tlsSecretsMetadata.keyName); - cursor.setLong(TlsSecretsMetadata.versionField, tlsSecretsMetadata.version); - curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - public Optional<TlsSecrets> getTlsSecrets(Optional<String> secretKeyname, ApplicationId applicationId) { - if (secretKeyname == null || secretKeyname.isEmpty()) { - return readTlsSecretsKeyFromZookeeper(applicationId); - } - return readFromSecretStore(secretKeyname); - } - - private Optional<TlsSecrets> readFromSecretStore(Optional<String> secretKeyname) { - if (secretKeyname.isEmpty()) return Optional.empty(); - try { - String cert = secretStore.getSecret(secretKeyname.get() + "-cert"); - String key = secretStore.getSecret(secretKeyname.get() + "-key"); - return Optional.of(new TlsSecrets(cert, key)); - } catch (RuntimeException e) { - // Assume not ready yet - return Optional.of(TlsSecrets.MISSING); - } - } - - private TlsSecrets readFromSecretStore(TlsSecretsMetadata tlsSecretsMetadata) { - try { - String cert = secretStore.getSecret(tlsSecretsMetadata.certName, tlsSecretsMetadata.version); - String key = secretStore.getSecret(tlsSecretsMetadata.keyName, tlsSecretsMetadata.version); - return new TlsSecrets(cert, key); - } catch (RuntimeException e) { - // Assume not ready yet - return TlsSecrets.MISSING; - } - } - - /** Returns a transaction which deletes these tls secrets key if they exist */ - public CuratorTransaction delete(ApplicationId application) { - if (!curator.exists(tlsSecretsKeyOf(application))) return CuratorTransaction.empty(curator); - return CuratorTransaction.from(CuratorOperations.delete(tlsSecretsKeyOf(application).getAbsolute()), curator); - } - - /** Returns the path storing the tls secrets key for an application */ - private Path tlsSecretsKeyOf(ApplicationId application) { - return path.append(application.serializedForm()); - } - - static class TlsSecretsMetadata { - final static String keyNameField = "keyName"; - final static String certNameField = "certName"; - final static String versionField = "version"; - String keyName; - String certName; - int version; - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index fb745bbb76b..d924d22cb39 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -266,13 +266,22 @@ public class ApplicationRepositoryTest { } { + PrepareResult prepareResult = deployApp(testApp); try { - deployApp(testApp); applicationRepository.delete(applicationId(), Duration.ZERO); fail("Should have gotten an exception"); } catch (InternalServerException e) { - assertEquals("Session 5 was not deleted (waited PT0S)", e.getMessage()); + assertEquals("test1.testapp was not deleted (waited PT0S), session " + prepareResult.sessionId(), e.getMessage()); } + + // No active session or remote session (deleted in step above), but an exception was thrown above + // A new delete should cleanup and be successful + LocalSession activeSession = applicationRepository.getActiveSession(applicationId()); + assertNull(activeSession); + Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); + assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId())); + + assertTrue(applicationRepository.delete(applicationId())); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java index 8a77b53875e..12f48778144 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java @@ -7,22 +7,26 @@ import java.util.HashMap; import java.util.Map; public class MockSecretStore implements SecretStore { - Map<String, String> secrets = new HashMap<>(); + Map<String, Map<Integer, String>> secrets = new HashMap<>(); @Override public String getSecret(String key) { if(secrets.containsKey(key)) - return secrets.get(key); + return secrets.get(key).get(0); throw new RuntimeException("Key not found: " + key); } @Override public String getSecret(String key, int version) { - return getSecret(key); + return secrets.get(key).get(version); + } + + public void put(String key, int version, String value) { + secrets.computeIfAbsent(key, k -> new HashMap<>()).put(version, value); } public void put(String key, String value) { - secrets.put(key, value); + put(key, 0, value); } public void remove(String key) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index a099db5ebe8..40115170b69 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -22,6 +22,11 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockReloadHandler; @@ -37,7 +42,8 @@ import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -46,9 +52,14 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.security.auth.x500.X500Principal; import java.io.File; import java.io.IOException; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -73,6 +84,9 @@ public class SessionPreparerTest { private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = new Version(1, 2, 3); private static final Version version321 = new Version(3, 2, 1); + private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; @@ -231,15 +245,37 @@ public class SessionPreparerTest { var tlskey = "vespa.tlskeys.tenant1--app1"; var applicationId = applicationId("test"); var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - secretStore.put(tlskey+"-cert", "CERT"); - secretStore.put(tlskey+"-key", "KEY"); + + secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); + prepare(new File("src/test/resources/deploy/hosted-app"), params); // Read from zk and verify cert and key are available - Optional<TlsSecrets> tlsSecrets = new TlsSecretsKeys(curator, tenantPath, secretStore).readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) + .readEndpointCertificateMetadata(applicationId) + .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void require_that_endpoint_certificate_metadata_is_written() throws IOException { + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); + secretStore.put("vespa.tlskeys.tenant1--app1-cert", 7, X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", 7, KeyUtils.toPem(keyPair.getPrivate())); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + // Read from zk and verify cert and key are available + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) + .readEndpointCertificateMetadata(applicationId) + .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); + + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); } @Test(expected = CertificateNotReadyException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java new file mode 100644 index 00000000000..d71eab25ce3 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java @@ -0,0 +1,90 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.config.server.MockSecretStore; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class EndpointCertificateMetadataStoreTest { + + private static final Path tenantPath = Path.createRoot(); + private static final Path endpointCertificateMetadataPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); + private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), + ApplicationName.from("test"), InstanceName.defaultName()); + + private MockCurator curator; + private MockSecretStore secretStore = new MockSecretStore(); + private EndpointCertificateMetadataStore endpointCertificateMetadataStore; + private EndpointCertificateRetriever endpointCertificateRetriever; + private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); + + @Before + public void setUp() { + curator = new MockCurator(); + endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); + endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); + + secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); + } + + @Test + public void reads_string_format() { + curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes()); + + // Read from zk and verify cert and key are available + var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void reads_object_format() { + curator.set(endpointCertificateMetadataPath, + "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" + .getBytes()); + + // Read from zk and verify cert and key are available + var secrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + assertTrue(secrets.isPresent()); + assertTrue(secrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(secrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void can_write_object_format() { + var endpointCertificateMetadata = new EndpointCertificateMetadata("key-name", "cert-name", 1); + + endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, endpointCertificateMetadata); + + assertEquals("{\"keyName\":\"key-name\",\"certName\":\"cert-name\",\"version\":1}", + new String(curator.getData(endpointCertificateMetadataPath).orElseThrow())); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java deleted file mode 100644 index c71c7b8e040..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.MockSecretStore; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TlsSecretsKeysTest { - - private static final Path tenantPath = Path.createRoot(); - private static final Path tlsSecretsKeysPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); - private static final String tlskey = "vespa.tlskeys.tenant1--app1"; - private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), - ApplicationName.from("test"), InstanceName.defaultName()); - - private MockCurator curator; - private MockSecretStore secretStore = new MockSecretStore(); - private TlsSecretsKeys tlsSecretsKeys; - - @Before - public void setUp() { - curator = new MockCurator(); - tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); - secretStore.put(tlskey + "-cert", "CERT"); - secretStore.put(tlskey + "-key", "KEY"); - } - - @Test - public void reads_string_format() { - curator.set(tlsSecretsKeysPath, ('"' + tlskey + '"').getBytes()); - - // Read from zk and verify cert and key are available - var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); - } - - @Test - public void reads_object_format() { - curator.set(tlsSecretsKeysPath, - "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" - .getBytes()); - - // Read from zk and verify cert and key are available - var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); - } - - @Test - public void can_write_object_format() { - var tlsSecretsMetadata = new TlsSecretsKeys.TlsSecretsMetadata(); - tlsSecretsMetadata.certName = "cert-name"; - tlsSecretsMetadata.keyName = "key-name"; - tlsSecretsMetadata.version = 1; - - tlsSecretsKeys.writeTlsSecretsMetadata(applicationId, tlsSecretsMetadata); - - assertEquals("{\"certName\":\"cert-name\",\"keyName\":\"key-name\",\"version\":1}", - new String(curator.getData(tlsSecretsKeysPath).get())); - } -} diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 5dc9a863970..91c0e4cc6bf 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -854,4 +854,4 @@ ], "fields": [] } -} +}
\ No newline at end of file diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 30cdbbfe531..a621545446f 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.25.v20191220</jetty.version> + <jetty.version>9.4.26.v20200117</jetty.version> <lz4.version>1.3.0</lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-search/src/main/java/com/yahoo/prelude/Ping.java b/container-search/src/main/java/com/yahoo/prelude/Ping.java index dd14e150d95..1d5d4c92827 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Ping.java +++ b/container-search/src/main/java/com/yahoo/prelude/Ping.java @@ -4,7 +4,7 @@ package com.yahoo.prelude; /** * A ping, typically to ask whether backend is alive. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class Ping { diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index a60fba9a4f7..1e5513f1274 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -4,82 +4,102 @@ package com.yahoo.prelude; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.statistics.ElapsedTime; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * An answer from Ping. * - * @author Steinar Knutsen + * @author bratseth */ public class Pong { - private String pingInfo=""; - private final List<ErrorMessage> errors = new ArrayList<>(1); - private ElapsedTime elapsed = new ElapsedTime(); + private final ElapsedTime elapsed = new ElapsedTime(); private final Optional<Long> activeDocuments; + private final boolean isBlockingWrites; + private final Optional<ErrorMessage> error; public Pong() { - this.activeDocuments = Optional.empty(); + this(Optional.empty(), false, Optional.empty()); } public Pong(ErrorMessage error) { - errors.add(error); - this.activeDocuments = Optional.empty(); + this(Optional.empty(), false, Optional.of(error)); } public Pong(long activeDocuments) { - this.activeDocuments = Optional.of(activeDocuments); + this(Optional.of(activeDocuments), false, Optional.empty()); } - public void addError(ErrorMessage error) { - errors.add(error); + public Pong(long activeDocuments, boolean isBlockingWrites) { + this(Optional.of(activeDocuments), isBlockingWrites, Optional.empty()); } - public ErrorMessage getError(int i) { - return errors.get(i); + private Pong(Optional<Long> activeDocuments, boolean isBlockingWrites, Optional<ErrorMessage> error) { + this.activeDocuments = activeDocuments; + this.isBlockingWrites = isBlockingWrites; + this.error = error; } - /** Returns the number of active documents in the backend responding in this Pong, if available */ - public Optional<Long> activeDocuments() { - return activeDocuments; + /** + * @deprecated do not use. Additional errors are ignored. + */ + @Deprecated + public void addError(ErrorMessage error) { } + + /** + * @deprecated use error() instead + */ + @Deprecated + public ErrorMessage getError(int i) { + if (i > 1) throw new IllegalArgumentException("No error at position " + i); + if (i == 0 && error.isEmpty()) throw new IllegalArgumentException("No error at position " + i); + return error.get(); } - /** Returns the number of nodes which responded to this Pong, if available */ + public Optional<ErrorMessage> error() { return error; } + + /** Returns the number of active documents in the backend responding in this Pong, if available */ + public Optional<Long> activeDocuments() { return activeDocuments; } + + /** Returns true if the pinged node is currently blocking write operations due to being full */ + public boolean isBlockingWrites() { return isBlockingWrites; } + + /** + * Returns Optional.empty() + * + * @return empty + * @deprecated do not use. There is always one pong per node. + */ + @Deprecated public Optional<Integer> activeNodes() { return Optional.empty(); } + /** + * Returns a list containing 0 or 1 errors + * + * @deprecated use error() instead + */ + @Deprecated public List<ErrorMessage> getErrors() { - return Collections.unmodifiableList(errors); + return error.stream().collect(Collectors.toList()); } /** Returns whether there is an error or not */ - public boolean badResponse() { - return ! errors.isEmpty(); - } + public boolean badResponse() { return error.isPresent(); } - public ElapsedTime getElapsedTime() { - return elapsed; - } + public ElapsedTime getElapsedTime() { return elapsed; } /** Returns a string which included the ping info (if any) and any errors added to this */ @Override public String toString() { - StringBuilder m = new StringBuilder("Result of pinging"); - if (pingInfo.length() > 0) { - m.append(" using "); - m.append(pingInfo); - } - if (errors.size() > 0) - m.append(" "); - for (int i = 0; i < errors.size(); i++) { - m.append(errors.get(i).toString()); - if ( i <errors.size()-1) - m.append(", "); - } + StringBuilder m = new StringBuilder("Ping result"); + activeDocuments.ifPresent(docCount -> m.append(" active docs: ").append(docCount)); + if (isBlockingWrites) + m.append(" blocking writes: true"); + error.ifPresent(e -> m.append(" error: ").append(error)); return m.toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java index 153606e6d99..69131b6c690 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java @@ -8,7 +8,7 @@ import java.nio.ByteBuffer; /** - * <p>A set of terms which must be near each other to match.</p> + * A set of terms which must be near each other to match. * * @author bratseth * @author havardpe @@ -18,7 +18,7 @@ public class NearItem extends CompositeItem { protected int distance; /** The default distance used if none is specified: 2 */ - public static final int defaultDistance=2; + public static final int defaultDistance = 2; /** Creates a NEAR item with distance 2 */ public NearItem() { @@ -26,8 +26,7 @@ public class NearItem extends CompositeItem { } /** - * Creates a <i>near</i> item with a limit to the distance - * between the words. + * Creates a <i>near</i> item with a limit to the distance between the words. * * @param distance the number of word position which may separate * the words for this near item to match @@ -47,14 +46,17 @@ public class NearItem extends CompositeItem { return distance; } + @Override public ItemType getItemType() { return ItemType.NEAR; } + @Override public String getName() { return "NEAR"; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); IntegerCompressor.putCompressedPositiveNumber(distance, buffer); @@ -67,6 +69,7 @@ public class NearItem extends CompositeItem { } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); @@ -75,6 +78,7 @@ public class NearItem extends CompositeItem { buffer.append(" "); } + @Override public int hashCode() { return super.hashCode() + 23* distance; } @@ -83,10 +87,11 @@ public class NearItem extends CompositeItem { * Returns whether this item is of the same class and * contains the same state as the given item */ + @Override public boolean equals(Object object) { if (!super.equals(object)) return false; NearItem other = (NearItem) object; // Ensured by superclass - if (this.distance !=other.distance) return false; + if (this.distance != other.distance) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java index 84e93d5de8f..88982195af6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java @@ -25,10 +25,12 @@ public class ONearItem extends NearItem { super(distance); } + @Override public ItemType getItemType() { return ItemType.ONEAR; } + @Override public String getName() { return "ONEAR"; } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 06cbf9a9706..20f56c86f7b 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -96,7 +96,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod future.cancel(true); if (pong.badResponse()) { - monitor.failed(p, pong.getError(0)); + monitor.failed(p, pong.error().get()); log(LogLevel.FINE, "Failed ping - ", pong); } else { monitor.responded(p); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index e54e2187818..bc0a38617ee 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -13,6 +13,7 @@ import java.util.Optional; * @author bratseth */ interface Client { + /** Creates a connection to a particular node in this */ NodeConnection createConnection(String hostname, int port); @@ -21,6 +22,7 @@ interface Client { } class ResponseOrError<T> { + final Optional<T> response; final Optional<String> error; @@ -93,6 +95,7 @@ interface Client { } class ProtobufResponse { + private final byte compression; private final int uncompressedSize; private final byte[] compressedPayload; @@ -114,6 +117,7 @@ interface Client { public byte[] compressedPayload() { return compressedPayload; } + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index f2e22ba86dc..e0f1dc5e675 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -18,6 +18,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; public class RpcPing implements Callable<Pong> { + private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; @@ -57,8 +58,7 @@ public class RpcPing implements Callable<Pong> { var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); - connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), - timeoutSeconds); + connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), timeoutSeconds); } private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { @@ -67,12 +67,13 @@ public class RpcPing implements Callable<Pong> { var reply = SearchProtocol.MonitorReply.parseFrom(responseBytes); if (reply.getDistributionKey() != node.key()) { - return new Pong(ErrorMessage.createBackendCommunicationError( - "Expected pong from node id " + node.key() + ", response is from id " + reply.getDistributionKey())); + return new Pong(ErrorMessage.createBackendCommunicationError("Expected pong from node id " + node.key() + + ", response is from id " + reply.getDistributionKey())); } else if (!reply.getOnline()) { return new Pong(ErrorMessage.createBackendCommunicationError("Node id " + node.key() + " reports being offline")); } else { - return new Pong(reply.getActiveDocs()); + return new Pong(reply.getActiveDocs(), reply.getIsBlockingWrites()); } } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index eff6ae26816..5f211c37917 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -274,7 +274,7 @@ public class SearchCluster implements NodeManager<Node> { futurePong.cancel(true); if (pong.badResponse()) { - clusterMonitor.failed(node, pong.getError(0)); + clusterMonitor.failed(node, pong.error().get()); } else { if (pong.activeDocuments().isPresent()) { node.setActiveDocuments(pong.activeDocuments().get()); diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java index 2992d8ab896..8dcc25e4b3b 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java @@ -56,11 +56,10 @@ public class ClusterSearcherTestCase { @Override public Pong ping(Ping ping, Execution execution) { - Pong pong = new Pong(); - if (isBlocking()) { - pong.addError(ErrorMessage.createTimeout("Dummy timeout")); - } - return new Pong(); + if (isBlocking()) + return new Pong(ErrorMessage.createTimeout("Dummy timeout")); + else + return new Pong(); } public boolean isBlocking() { diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index c90d2774bd1..a824edd1996 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -182,10 +182,10 @@ public class ClusteredConnectionTestCase { @Override public Pong ping(Ping ping,Connection connection) { - Pong pong = new Pong(); - if (connection.getResponse() == null) - pong.addError(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); - return pong; + if (connection.getResponse() != null) + return new Pong(); + else + return new Pong(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 74e4ac9d404..a009f002954 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.InputStream; @@ -87,4 +88,8 @@ public interface ConfigServer { /** List all flag data for the given zone */ List<FlagData> listFlagData(ZoneId zone); + /** Gets status for tester application */ + // TODO: Remove default implementation when implemented in internal repo + default TesterCloud.Status getTesterStatus(DeploymentId deployment) { return TesterCloud.Status.SUCCESS; } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 685eed22699..bbe533db9b5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -51,8 +51,6 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { if (commit.isPresent() && commit.get().length() > 128) throw new IllegalArgumentException("Commit may not be longer than 128 characters"); - sourceUrl.map(URI::create); - if (authorEmail.isPresent() && ! authorEmail.get().matches("[^@]+@[^@]+")) throw new IllegalArgumentException("Invalid author email '" + authorEmail.get() + "'."); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 4f6fe2ac2db..d3e21f0d399 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; @@ -70,6 +71,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; private final MavenRepository mavenRepository; + private final Metric metric; /** * Creates a controller @@ -77,22 +79,15 @@ public class Controller extends AbstractComponent implements ApplicationIdSource * @param curator the curator instance storing the persistent state of the controller. */ @Inject - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - AccessControl accessControl, - FlagSource flagSource, - MavenRepository mavenRepository, - ServiceRegistry serviceRegistry) { - this(curator, rotationsConfig, - accessControl, - com.yahoo.net.HostName::getLocalhost, flagSource, - mavenRepository, serviceRegistry); + public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, + MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric) { + this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, + mavenRepository, serviceRegistry, metric); } - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - AccessControl accessControl, - Supplier<String> hostnameSupplier, - FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry) { + public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, + Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, + ServiceRegistry serviceRegistry, Metric metric) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -101,7 +96,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); - + this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); metrics = new ConfigServerMetrics(serviceRegistry.configServer()); nameServiceForwarder = new NameServiceForwarder(curator); @@ -265,6 +260,10 @@ public class Controller extends AbstractComponent implements ApplicationIdSource return auditLogger; } + public Metric metric() { + return metric; + } + private Set<CloudName> clouds() { return zoneRegistry.zones().all().zones().stream() .map(ZoneApi::getCloudName) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 51841396d6a..35d478fb332 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -196,6 +196,13 @@ public class DeploymentTrigger { instance -> instance.withJobPause(jobType, OptionalLong.of(until.toEpochMilli()))))); } + /** Resumes a previously paused job, letting it be triggered normally. */ + public void resumeJob(ApplicationId id, JobType jobType) { + applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> + applications().store(application.with(id.instance(), + instance -> instance.withJobPause(jobType, OptionalLong.empty())))); + } + /** Triggers a change of this application, unless it already has a change. */ public void triggerChange(ApplicationId instanceId, Change change) { applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index bd61d85fbc0..9d09394a571 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -17,6 +17,9 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -469,7 +472,7 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> endTests(RunId id, DualLogger logger) { - if ( ! deployment(id.application(), id.type()).isPresent()) { + if (deployment(id.application(), id.type()).isEmpty()) { logger.log(INFO, "Deployment expired before tests could complete."); return Optional.of(aborted); } @@ -485,15 +488,22 @@ public class InternalStepRunner implements StepRunner { } } - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if ( ! testerEndpoint.isPresent()) { - logger.log("Endpoints for tester not found -- trying again later."); - return Optional.empty(); - } - controller.jobController().updateTestLog(id); - TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); + BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource()); + ZoneId zoneId = id.type().zone(controller.system()); + TesterCloud.Status testStatus; + if (useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value()) { + testStatus = controller.serviceRegistry().configServer().getTesterStatus(new DeploymentId(id.application(), zoneId)); + } else { + Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); + if (testerEndpoint.isEmpty()) { + logger.log("Endpoints for tester not found -- trying again later."); + return Optional.empty(); + } + testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); + } + switch (testStatus) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 702d3ae9292..c8cfc8ac286 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -79,6 +80,7 @@ public class JobController { private final BufferedLogStore logs; private final TesterCloud cloud; private final Badges badges; + private final JobMetrics metric; private AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); @@ -88,6 +90,7 @@ public class JobController { this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); this.cloud = controller.serviceRegistry().testerCloud(); this.badges = new Badges(controller.zoneRegistry().badgeUrl()); + this.metric = new JobMetrics(controller.metric(), controller.system()); } public TesterCloud cloud() { return cloud; } @@ -360,6 +363,7 @@ public class JobController { } }); logs.flush(id); + metric.jobFinished(run.id().job(), finishedRun.status()); return finishedRun; }); } @@ -416,6 +420,7 @@ public class JobController { RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant())); + metric.jobStarted(newId.job()); }); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java new file mode 100644 index 00000000000..a6ffb56492f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -0,0 +1,64 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; + +import java.util.Map; + +/** + * Records metrics related to deployment jobs. + * + * @author jonmv + */ +public class JobMetrics { + + public static final String start = "deployment.start"; + public static final String outOfCapacity = "deployment.outOfCapacity"; + public static final String deploymentFailure = "deployment.deploymentFailure"; + public static final String convergenceFailure = "deployment.convergenceFailure"; + public static final String testFailure = "deployment.testFailure"; + public static final String error = "deployment.error"; + public static final String abort = "deployment.abort"; + public static final String success = "deployment.success"; + + private final Metric metric; + private final SystemName system; + + public JobMetrics(Metric metric, SystemName system) { + this.metric = metric; + this.system = system; + } + + public void jobStarted(JobId id) { + metric.add(start, 1, metric.createContext(contextOf(id))); + } + + public void jobFinished(JobId id, RunStatus status) { + metric.add(valueOf(status), 1, metric.createContext(contextOf(id))); + } + + Map<String, String> contextOf(JobId id) { + return Map.of("tenant", id.application().tenant().value(), + "application", id.application().application().value(), + "instance", id.application().instance().value(), + "job", id.type().jobName(), + "environment", id.type().environment().value(), + "region", id.type().zone(system).region().value()); + } + + static String valueOf(RunStatus status) { + switch (status) { + case outOfCapacity: return outOfCapacity; + case deploymentFailed: return deploymentFailure; + case installationFailed: return convergenceFailure; + case testFailure: return testFailure; + case error: return error; + case aborted: return abort; + case success: return success; + default: throw new IllegalArgumentException("Unexpected run status '" + status + "'"); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7aa6fe11ac4..f6cf776cbfa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -296,6 +296,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return resume(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -690,6 +691,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse(type.jobName() + " for " + id + " paused for " + DeploymentTrigger.maxPause); } + private HttpResponse resume(ApplicationId id, JobType type) { + controller.applications().deploymentTrigger().resumeJob(id, type); + return new MessageResponse(type.jobName() + " for " + id + " resumed"); + } + private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); @@ -1997,6 +2003,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<String> sourceUrl = optional("sourceUrl", submitOptions); Optional<String> authorEmail = optional("authorEmail", submitOptions); + sourceUrl.map(URI::create).ifPresent(url -> { + if (url.getHost() == null || url.getScheme() == null) + throw new IllegalArgumentException("Source URL must include scheme and host"); + }); + ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), true); controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index b564e5e5dcf..c83463bc1ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -391,7 +392,8 @@ public final class ControllerTester { () -> "test-controller", new InMemoryFlagSource(), new MockMavenRepository(), - serviceRegistry); + serviceRegistry, + new MetricsMock()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index b37d3d340cb..fef8ab32d17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -269,6 +270,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return List.of(); } + @Override + public TesterCloud.Status getTesterStatus(DeploymentId deployment) { + return TesterCloud.Status.SUCCESS; + } + public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { this.loadBalancers.putIfAbsent(zone, new ArrayList<>()); this.loadBalancers.get(zone).addAll(loadBalancers); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 21e4735f7bf..9de0020ce4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -11,12 +11,14 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.JobMetrics; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Test; import java.time.Duration; @@ -43,6 +45,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -59,6 +62,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -348,7 +352,44 @@ public class JobRunnerTest { jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); runner.run(); - assertTrue(jobs.last(id, systemTest).get().status() == aborted); + assertSame(aborted, jobs.last(id, systemTest).get().status()); + } + + @Test + public void jobMetrics() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + inThreadExecutor(), mappedRunner(outcomes)); + + TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); + ApplicationId id = appId.defaultInstance(); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + + for (RunStatus status : RunStatus.values()) { + if (status == success) continue; // Status not used for steps. + outcomes.put(deployTester, status); + jobs.start(id, systemTest, versions); + runner.run(); + jobs.finish(jobs.last(id, systemTest).get().id()); + } + + Map<String, String> context = Map.of("tenant", "tenant", + "application", "real", + "instance", "default", + "job", "system-test", + "environment", "test", + "region", "us-east-1"); + MetricsMock metric = ((MetricsMock) tester.controller().metric()); + assertEquals(RunStatus.values().length - 1, metric.getMetric(context::equals, JobMetrics.start).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.abort).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.error).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.success).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } public static ExecutorService inThreadExecutor() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index efb1ce4d5eb..96681dc1c8b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -520,6 +520,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"production-us-west-1 for tenant1.application1.instance1 paused for " + DeploymentTrigger.maxPause + "\"}"); + // DELETE a pause of a production job + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/pause", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"production-us-west-1 for tenant1.application1.instance1 resumed\"}"); + // POST a triggering to the same production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", POST) .userIdentity(USER_ID), diff --git a/eval/src/tests/ann/sift_benchmark.cpp b/eval/src/tests/ann/sift_benchmark.cpp index f20df926f24..dcfe1cf9c5c 100644 --- a/eval/src/tests/ann/sift_benchmark.cpp +++ b/eval/src/tests/ann/sift_benchmark.cpp @@ -285,6 +285,15 @@ TEST("require that HNSW via NNS api mostly works") { #endif +/** + * Before running the benchmark the ANN_SIFT1M data set must be downloaded and extracted: + * wget ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz + * tar -xf sift.tar.gz + * + * The benchmark program will load the data set from $HOME/sift if no directory is specified. + * + * More information about the dataset is found here: http://corpus-texmex.irisa.fr/. + */ int main(int argc, char **argv) { TEST_MASTER.init(__FILE__); std::string sift_dir = "."; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index eff8b48798e..9535b62b15c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -220,6 +220,12 @@ public class Flags { "Takes effect immediately", ZONE_ID, HOSTNAME); + public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( + "generate-l4-routing-config", false, + "Whether routing nodes should generate L4 routing config", + "Takes effect immediately", + ZONE_ID, HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/searchlib/src/protobuf/search_protocol.proto b/searchlib/src/protobuf/search_protocol.proto index 6a4d3cfb07a..ded19fe132f 100644 --- a/searchlib/src/protobuf/search_protocol.proto +++ b/searchlib/src/protobuf/search_protocol.proto @@ -88,4 +88,5 @@ message MonitorReply { bool online = 1; int64 active_docs = 2; int32 distribution_key = 3; + bool is_blocking_writes = 4; } diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 97b6cc344e1..cefa8ab2f51 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -19,11 +19,16 @@ import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; import java.security.GeneralSecurityException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Random; import static com.yahoo.security.Extension.SUBJECT_ALTERNATIVE_NAMES; import static java.util.stream.Collectors.toList; @@ -140,4 +145,20 @@ public class X509CertificateUtils { } } + public static boolean privateKeyMatchesPublicKey(PrivateKey privateKey, PublicKey publicKey) { + byte[] someRandomData = new byte[64]; + new Random().nextBytes(someRandomData); + + Signature signer = SignatureUtils.createSigner(privateKey); + Signature verifier = SignatureUtils.createVerifier(publicKey); + try { + signer.update(someRandomData); + verifier.update(someRandomData); + byte[] signature = signer.sign(); + return verifier.verify(signature); + } catch (SignatureException e) { + throw new RuntimeException(e); + } + } + } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index 76a93028efe..b4eca8328c1 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -17,7 +17,9 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -71,4 +73,18 @@ public class X509CertificateUtilsTest { assertThat(sans.size(), is(1)); assertThat(sans.get(0), equalTo(san)); } + + @Test + public void verifies_matching_cert_and_key() { + KeyPair ecKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + KeyPair ecKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + KeyPair rsaKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); + KeyPair rsaKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); + + assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairA.getPublic())); + assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairA.getPublic())); + + assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairB.getPublic())); + assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairB.getPublic())); + } }
\ No newline at end of file diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index 25b652b7312..05fb985dbaf 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -109,4 +109,4 @@ ], "fields": [] } -} +}
\ No newline at end of file diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index d6be27a0919..7c5b1ae319a 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.function.Function; +import java.util.logging.Logger; /** * Curator interface for Vespa. @@ -47,6 +48,8 @@ import java.util.function.Function; */ public class Curator implements AutoCloseable { + private static final Logger logger = Logger.getLogger(Curator.class.getName()); + private static final int ZK_SESSION_TIMEOUT = 30000; private static final int ZK_CONNECTION_TIMEOUT = 30000; @@ -115,7 +118,7 @@ public class Curator implements AutoCloseable { if (this.curatorFramework != null) { validateConnectionSpec(connectionSpec); validateConnectionSpec(zooKeeperEnsembleConnectionSpec); - addFakeListener(); + addLoggingListener(); curatorFramework.start(); } @@ -198,10 +201,13 @@ public class Curator implements AutoCloseable { return new InterProcessMutex(curatorFramework, lockPath); } - // To avoid getting warning in log, see ticket 6389740 - private void addFakeListener() { + private void addLoggingListener() { curatorFramework.getConnectionStateListenable().addListener((curatorFramework, connectionState) -> { - // empty, not needed now + switch (connectionState) { + case SUSPENDED: logger.info("ZK connection state change: SUSPENDED"); break; + case RECONNECTED: logger.info("ZK connection state change: RECONNECTED"); break; + case LOST: logger.warning("ZK connection state change: LOST"); break; + } }); } |