diff options
271 files changed, 3938 insertions, 4282 deletions
diff --git a/client/README.md b/client/README.md index ddea3591e38..d08b19a1a12 100644 --- a/client/README.md +++ b/client/README.md @@ -1,12 +1,12 @@ # vespa_query_dsl This lib is used for composing vespa YQL queries -referece: https://docs.vespa.ai/documentation/reference/query-language-reference.html +Reference: https://docs.vespa.ai/documentation/reference/query-language-reference.html # usage -please refer the unit test: +Please refer to the unit test: -https://github.com/vespa-engine/vespa/blob/master/client/src/test/groovy/com/yahoo/vespa/client/dsl/QTest.groovy +https://github.com/vespa-engine/vespa/tree/master/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy # todos - [ ] support `predicate` (https://docs.vespa.ai/documentation/predicate-fields.html) diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 6b466c65cdb..0f5a5e6271d 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -605,5 +605,511 @@ "public static final com.yahoo.config.application.api.ValidationOverrides empty", "public static final com.yahoo.config.application.api.ValidationOverrides all" ] + }, + "com.yahoo.config.model.api.ApplicationInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.config.provision.ApplicationId, long, com.yahoo.config.model.api.Model)", + "public com.yahoo.config.provision.ApplicationId getApplicationId()", + "public long getGeneration()", + "public com.yahoo.config.model.api.Model getModel()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigChangeAction$Type": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ConfigChangeAction$Type[] values()", + "public static com.yahoo.config.model.api.ConfigChangeAction$Type valueOf(java.lang.String)", + "public java.lang.String toString()" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type RESTART", + "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type REFEED" + ] + }, + "com.yahoo.config.model.api.ConfigChangeAction": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public abstract java.lang.String getMessage()", + "public abstract java.util.List getServices()", + "public abstract boolean allowed()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigChangeRefeedAction": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public java.lang.String name()", + "public abstract java.lang.String getDocumentType()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigChangeRestartAction": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.model.api.ConfigChangeAction" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()", + "public boolean allowed()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigDefinitionRepo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.util.Map getConfigDefinitions()", + "public abstract com.yahoo.vespa.config.buildergen.ConfigDefinition get(com.yahoo.vespa.config.ConfigDefinitionKey)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigDefinitionStore": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.vespa.config.ConfigDefinition getConfigDefinition(com.yahoo.vespa.config.ConfigDefinitionKey)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigModelPlugin": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [], + "fields": [] + }, + "com.yahoo.config.model.api.ConfigServerSpec": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract java.lang.String getHostName()", + "public abstract int getConfigServerPort()", + "public int getHttpPort()", + "public abstract int getZooKeeperPort()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ContainerEndpoint": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.util.List)", + "public java.lang.String clusterId()", + "public java.util.List names()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.EndpointCertificateMetadata": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String, int)", + "public java.lang.String keyName()", + "public java.lang.String certName()", + "public int version()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.EndpointCertificateSecrets": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String)", + "public java.lang.String certificate()", + "public java.lang.String key()", + "public boolean isMissing()" + ], + "fields": [ + "public static final com.yahoo.config.model.api.EndpointCertificateSecrets MISSING" + ] + }, + "com.yahoo.config.model.api.FileDistribution": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void startDownload(java.lang.String, int, java.util.Set)", + "public abstract java.io.File getFileReferencesDir()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.HostInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.util.Collection)", + "public java.lang.String getHostname()", + "public java.util.Collection getServices()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.HostProvisioner": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.provision.HostSpec allocateHost(java.lang.String)", + "public abstract java.util.List prepare(com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, int, com.yahoo.config.provision.ProvisionLogger)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.Model": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.vespa.config.ConfigPayload getConfig(com.yahoo.vespa.config.ConfigKey, com.yahoo.vespa.config.buildergen.ConfigDefinition)", + "public abstract java.util.Set allConfigsProduced()", + "public abstract java.util.Collection getHosts()", + "public abstract java.util.Set allConfigIds()", + "public abstract void distributeFiles(com.yahoo.config.model.api.FileDistribution)", + "public abstract java.util.Set fileReferences()", + "public abstract com.yahoo.config.provision.AllocatedHosts allocatedHosts()", + "public boolean allowModelVersionMismatch(java.time.Instant)", + "public boolean skipOldConfigModels(java.time.Instant)", + "public com.yahoo.component.Version version()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelContext$Properties": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract boolean multitenant()", + "public abstract com.yahoo.config.provision.ApplicationId applicationId()", + "public abstract java.util.List configServerSpecs()", + "public abstract com.yahoo.config.provision.HostName loadBalancerName()", + "public abstract java.net.URI ztsUrl()", + "public abstract java.lang.String athenzDnsSuffix()", + "public abstract boolean hostedVespa()", + "public abstract com.yahoo.config.provision.Zone zone()", + "public abstract java.util.Set endpoints()", + "public abstract boolean isBootstrap()", + "public abstract boolean isFirstTimeDeployment()", + "public boolean useDedicatedNodeForLogserver()", + "public abstract boolean useAdaptiveDispatch()", + "public java.util.Optional tlsSecrets()", + "public java.util.Optional endpointCertificateSecrets()", + "public abstract double defaultTermwiseLimit()", + "public abstract boolean useBucketSpaceMetric()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelContext": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.application.api.ApplicationPackage applicationPackage()", + "public abstract java.util.Optional previousModel()", + "public abstract java.util.Optional permanentApplicationPackage()", + "public abstract java.util.Optional hostProvisioner()", + "public abstract com.yahoo.config.application.api.DeployLogger deployLogger()", + "public abstract com.yahoo.config.model.api.ConfigDefinitionRepo configDefinitionRepo()", + "public abstract com.yahoo.config.application.api.FileRegistry getFileRegistry()", + "public abstract com.yahoo.config.model.api.ModelContext$Properties properties()", + "public java.util.Optional appDir()", + "public abstract com.yahoo.component.Version modelVespaVersion()", + "public abstract com.yahoo.component.Version wantedNodeVespaVersion()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelCreateResult": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.config.model.api.Model, java.util.List)", + "public com.yahoo.config.model.api.Model getModel()", + "public java.util.List getConfigChangeActions()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelFactory": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.component.Version version()", + "public abstract com.yahoo.config.model.api.Model createModel(com.yahoo.config.model.api.ModelContext)", + "public abstract com.yahoo.config.model.api.ModelCreateResult createAndValidateModel(com.yahoo.config.model.api.ModelContext, com.yahoo.config.model.api.ValidationParameters)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ModelState": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.config.model.api.Model getModel()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.PortInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(int, java.util.Collection)", + "public int getPort()", + "public java.util.Collection getTags()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.ServiceInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String, java.util.Collection, java.util.Map, java.lang.String, java.lang.String)", + "public java.lang.String getServiceName()", + "public java.lang.String getConfigId()", + "public java.lang.String getServiceType()", + "public java.util.Optional getProperty(java.lang.String)", + "public java.util.Collection getPorts()", + "public java.lang.String getHostName()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.SuperModel": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(java.util.Map)", + "public java.util.Map getModelsPerTenant()", + "public java.util.Map getModels()", + "public java.util.List getAllApplicationInfos()", + "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)", + "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)", + "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.SuperModelListener": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)", + "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)" + ], + "fields": [] + }, + "com.yahoo.config.model.api.SuperModelProvider": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void registerListener(com.yahoo.config.model.api.SuperModelListener)", + "public abstract com.yahoo.config.model.api.SuperModel getSuperModel()" + ], + "fields": [] + }, + "com.yahoo.config.model.api.TlsSecrets": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String, java.lang.String)", + "public void <init>(com.yahoo.config.model.api.EndpointCertificateSecrets)", + "public java.lang.String certificate()", + "public java.lang.String key()", + "public boolean isMissing()" + ], + "fields": [ + "public static final com.yahoo.config.model.api.TlsSecrets MISSING" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$CheckRouting": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors[] values()", + "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors TRUE", + "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors FALSE" + ] + }, + "com.yahoo.config.model.api.ValidationParameters": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors)", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$CheckRouting)", + "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors, com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange, com.yahoo.config.model.api.ValidationParameters$CheckRouting)", + "public boolean ignoreValidationErrors()", + "public boolean failOnIncompatibleChanges()", + "public boolean checkRouting()" + ], + "fields": [] } }
\ No newline at end of file diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java new file mode 100644 index 00000000000..a1fae9bb148 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateMetadata.java @@ -0,0 +1,35 @@ +package com.yahoo.config.model.api; + +public class EndpointCertificateMetadata { + + private final String keyName; + private final String certName; + private final int version; + + public EndpointCertificateMetadata(String keyName, String certName, int version) { + this.keyName = keyName; + this.certName = certName; + this.version = version; + } + + public String keyName() { + return keyName; + } + + public String certName() { + return certName; + } + + public int version() { + return version; + } + + @Override + public String toString() { + return "EndpointCertificateMetadata{" + + "keyName='" + keyName + '\'' + + ", certName='" + certName + '\'' + + ", version=" + version + + '}'; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java new file mode 100644 index 00000000000..6fcbac4f422 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/EndpointCertificateSecrets.java @@ -0,0 +1,30 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +public class EndpointCertificateSecrets { + public static final EndpointCertificateSecrets MISSING = new EndpointCertificateSecrets(); + + private final String certificate; + private final String key; + + private EndpointCertificateSecrets() { + this(null, null); + } + + public EndpointCertificateSecrets(String certificate, String key) { + this.certificate = certificate; + this.key = key; + } + + public String certificate() { + return certificate; + } + + public String key() { + return key; + } + + public boolean isMissing() { + return this == MISSING; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 323aa473580..81ac02a5400 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -54,8 +54,9 @@ public interface ModelContext { // TODO: Remove when Vespa 7.112 is the oldest config model in use default boolean useDedicatedNodeForLogserver() { return true; } boolean useAdaptiveDispatch(); - // TODO: Remove temporary default implementation + // TODO: Remove temporary default implementations default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } + default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } double defaultTermwiseLimit(); boolean useBucketSpaceMetric(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java index 6a8b5a237ab..0937b8b77ec 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/TlsSecrets.java @@ -16,6 +16,11 @@ public class TlsSecrets { this.key = key; } + public TlsSecrets(EndpointCertificateSecrets endpointCertificateSecrets) { + this.certificate = endpointCertificateSecrets.certificate(); + this.key = endpointCertificateSecrets.key(); + } + public String certificate() { return certificate; } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index 52ce35a19fb..a3478026520 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -1,5 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi // Not really "public", only annotated as such to enable the ABI checker plugin package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.api.annotations.PublicApi; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index b286b94c699..7c9e930bb4f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -15,7 +15,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -255,7 +255,7 @@ public class DeployState implements ConfigDefinitionStore { public Instant now() { return now; } - public Optional<TlsSecrets> tlsSecrets() { return properties.tlsSecrets(); } + public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); } public Optional<String> tlsClientAuthority() { var caFile = applicationPackage.getClientSecurityFile(); @@ -289,7 +289,6 @@ public class DeployState implements ConfigDefinitionStore { private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; - private Optional<TlsSecrets> tlsSecrets = Optional.empty(); public Builder applicationPackage(ApplicationPackage applicationPackage) { this.applicationPackage = applicationPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9d561a79c75..9f4d1b09f91 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -39,7 +40,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useDedicatedNodeForLogserver = false; private boolean useAdaptiveDispatch = false; private double defaultTermwiseLimit = 1.0; - private Optional<TlsSecrets> tlsSecrets = Optional.empty(); + private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); @Override public boolean multitenant() { return multitenant; } @@ -56,7 +57,8 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } - @Override public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } + @Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useBucketSpaceMetric() { return true; } @@ -95,9 +97,8 @@ public class TestProperties implements ModelContext.Properties { return this; } - - public TestProperties setTlsSecrets(Optional<TlsSecrets> tlsSecrets) { - this.tlsSecrets = tlsSecrets; + public TestProperties setEndpointCertificateSecrets(Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { + this.endpointCertificateSecrets = endpointCertificateSecrets; return this; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 58ef47b7ba9..441d1b5b8df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -69,7 +69,7 @@ public class RankProfileTypeSettingsProcessor extends Processor { } private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) { - for (RankProfile profile : rankProfileRegistry.all()) { + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { profile.addAttributeType(attributeName, attributeType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java index 32ecc6e36ba..21fa05ceeab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsNodesConfigGenerator.java @@ -22,7 +22,7 @@ public class MetricsNodesConfigGenerator { private static MetricsNodesConfig.Node.Builder toNodeBuilder(MetricsProxyContainer container) { var builder = new MetricsNodesConfig.Node.Builder() - .nodeId(container.getHost().getConfigId()) + .role(container.getHost().getConfigId()) .hostname(container.getHostName()) .metricsPort(MetricsProxyContainer.BASEPORT) .metricsPath(MetricsV1Handler.VALUES_PATH); @@ -30,7 +30,7 @@ public class MetricsNodesConfigGenerator { if (container.isHostedVespa) container.getHostResource().spec().membership() .map(ClusterMembership::stringValue) - .ifPresent(builder::nodeId); + .ifPresent(builder::role); return builder; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java index 2f972b8ecb3..f00ad0f0dbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java @@ -1,17 +1,17 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.vespa.model.VespaModel; -public class TlsSecretsValidator extends Validator { +public class EndpointCertificateSecretsValidator extends Validator { /** This check is delayed until validation to allow node provisioning to complete while we are waiting for cert */ @Override public void validate(VespaModel model, DeployState deployState) { - if (deployState.tlsSecrets().isPresent() && deployState.tlsSecrets().get() == TlsSecrets.MISSING) { + if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) { throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 8eabc61f71f..1e4a45428b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -57,7 +57,7 @@ public class Validation { new DeploymentSpecValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index f14ad9c51a5..8fdcf249bbc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.model.VespaModel; import java.time.Instant; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -60,7 +61,7 @@ public class ResourcesReductionValidator implements ChangeValidator { private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) { // don't allow more than 50% reduction, but always allow to reduce by 1 if (nextValue >= currentValue * 0.5 || nextValue >= currentValue - 1) return Optional.empty(); - return Optional.of(String.format("Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue)); + return Optional.of(String.format(Locale.ENGLISH ,"Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue)); } private static Map<Pair<ClusterSpec.Type, ClusterSpec.Id>, NodeResources> getRequestedResourcesByClusterId(VespaModel vespaModel) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 5e0dde6161d..efd00528d54 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -5,7 +5,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.BundlesConfig; @@ -55,7 +54,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private ContainerModelEvaluation modelEvaluation; - private Optional<TlsSecrets> tlsSecrets; private Optional<String> tlsClientAuthority; private MbusParams mbusParams; @@ -65,8 +63,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState); - - this.tlsSecrets = deployState.tlsSecrets(); this.tlsClientAuthority = deployState.tlsClientAuthority(); restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); servletGroup = new ConfigProducerGroup<>(this, "servlet"); @@ -205,10 +201,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } } - public Optional<TlsSecrets> getTlsSecrets() { - return tlsSecrets; - } - public Optional<String> getTlsClientAuthority() { return tlsClientAuthority; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 7a08a3c1a7b..12db3b87243 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -23,15 +23,15 @@ public class HostedSslConnectorFactory extends ConnectorFactory { /** * Create connector factory that uses a certificate provided by the config-model / configserver. */ - public static HostedSslConnectorFactory withProvidedCertificate(String serverName, TlsSecrets tlsSecrets) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, /*tlsCaCertificates*/null), false); + public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificates*/null), false); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ - public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { - return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, tlsSecrets, tlsCaCertificates), true); + public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, tlsCaCertificates), true); } /** @@ -47,11 +47,11 @@ public class HostedSslConnectorFactory extends ConnectorFactory { } private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider( - String serverName, TlsSecrets tlsSecrets, String tlsCaCertificates) { + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { return new ConfiguredDirectSslProvider( serverName, - tlsSecrets.key(), - tlsSecrets.certificate(), + endpointCertificateSecrets.key(), + endpointCertificateSecrets.certificate(), /*caCertificatePath*/null, tlsCaCertificates, ClientAuth.Enum.WANT_AUTH); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3da0b01f614..aef2697a5dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -13,7 +13,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; @@ -327,15 +327,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String serverName = server.getComponentId().getName(); // If the deployment contains certificate/private key reference, setup TLS port - if (deployState.tlsSecrets().isPresent()) { + if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); } - TlsSecrets tlsSecrets = deployState.tlsSecrets().get(); + EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, tlsSecrets, deployState.tlsClientAuthority().get()) - : HostedSslConnectorFactory.withProvidedCertificate(serverName, tlsSecrets); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets); server.addConnector(connectorFactory); } else { server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); diff --git a/config-model/src/test/derived/tensor2/first.sd b/config-model/src/test/derived/tensor2/first.sd new file mode 100644 index 00000000000..80554572503 --- /dev/null +++ b/config-model/src/test/derived/tensor2/first.sd @@ -0,0 +1,8 @@ +search first { + document first { + field first_field type tensor(first[10]) { + indexing: summary | attribute + } + } +} + diff --git a/config-model/src/test/derived/tensor2/rank-profiles.cfg b/config-model/src/test/derived/tensor2/rank-profiles.cfg new file mode 100644 index 00000000000..7e087832042 --- /dev/null +++ b/config-model/src/test/derived/tensor2/rank-profiles.cfg @@ -0,0 +1,14 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "vespa.type.attribute.second_field" +rankprofile[].fef.property[].value "tensor(second[10])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.second_field" +rankprofile[].fef.property[].value "tensor(second[10])" diff --git a/config-model/src/test/derived/tensor2/second.sd b/config-model/src/test/derived/tensor2/second.sd new file mode 100644 index 00000000000..ace0540c8bd --- /dev/null +++ b/config-model/src/test/derived/tensor2/second.sd @@ -0,0 +1,7 @@ +search second { + document second { + field second_field type tensor(second[10]) { + indexing: summary | attribute + } + } +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index ebd2c752d5e..61065cd4bcc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -138,4 +139,15 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("tensor"); } + @Test + public void testTensor2() throws IOException, ParseException { + String dir = "src/test/derived/tensor2/"; + SearchBuilder builder = new SearchBuilder(); + builder.importFile(dir + "first.sd"); + builder.importFile(dir + "second.sd"); + builder.build(); + derive("tensor2", builder, builder.getSearch("second")); + assertCorrectConfigFiles("tensor2"); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 3d43f8921a3..6fe69ac5c64 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -307,7 +307,7 @@ public class MetricsProxyContainerClusterTest { } private void assertNodeConfig(MetricsNodesConfig.Node node) { - assertTrue(node.nodeId().startsWith("container/foo/0/")); + assertTrue(node.role().startsWith("container/foo/0/")); assertTrue(node.hostname().startsWith("node-1-3-9-")); assertEquals(MetricsProxyContainer.BASEPORT, node.metricsPort()); assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java index cdb4ce955e2..21df39ebde8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/TlsSecretsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; @@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue; /** * @author andreer */ -public class TlsSecretsValidatorTest { +public class EndpointCertificateSecretsValidatorTest { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); @@ -43,21 +43,21 @@ public class TlsSecretsValidatorTest { @Test public void missing_certificate_fails_validation() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(TlsSecrets.MISSING)); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(EndpointCertificateSecrets.MISSING)); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); exceptionRule.expect(CertificateNotReadyException.class); exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet"); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } @Test public void validation_succeeds_with_certificate() throws Exception { - DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new TlsSecrets("cert", "key"))); + DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.of(new EndpointCertificateSecrets("cert", "key"))); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } @Test @@ -65,10 +65,10 @@ public class TlsSecretsValidatorTest { DeployState deployState = deployState(servicesXml(), deploymentXml(), Optional.empty()); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new TlsSecretsValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); } - private static DeployState deployState(String servicesXml, String deploymentXml, Optional<TlsSecrets> tlsSecrets) { + private static DeployState deployState(String servicesXml, String deploymentXml, Optional<EndpointCertificateSecrets> endpointCertificateSecretsSecrets) { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) .withDeploymentSpec(deploymentXml) @@ -79,7 +79,7 @@ public class TlsSecretsValidatorTest { .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(tlsSecrets)); + .setEndpointCertificateSecrets(endpointCertificateSecretsSecrets)); final DeployState deployState = builder.build(); assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 54d1c1c9793..1bbc4ea2684 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -5,7 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -693,7 +693,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) .zone(new Zone(SystemName.Public, Environment.prod, RegionName.defaultName())) .build(); createModel(root, state, null, clusterElem); @@ -772,13 +772,13 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatProvidingTlsSecretOpensPort4443() { + public void requireThatProvidingEndpointCertificateSecretsOpensPort4443() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", nodesXml, "</container>" ); - DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))).build(); + DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true).setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))).build(); createModel(root, state, null, clusterElem); ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 863781073f8..68f507c810d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -258,7 +258,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas .properties( new TestProperties() .setHostedVespa(true) - .setTlsSecrets(Optional.of(new TlsSecrets("CERT", "KEY")))) + .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY")))) .modelHostProvisioner(new HostsXmlProvisioner(new StringReader(hostsxml))) .build(); MockRoot root = new MockRoot("root", deployState); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index d2f26738301..3af7c7fdacc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -21,11 +21,14 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; @@ -74,6 +77,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -109,6 +113,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final Orchestrator orchestrator; private final LogRetriever logRetriever; private final TesterClient testerClient; + private final Metric metric; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -118,7 +123,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye HttpProxy httpProxy, ConfigserverConfig configserverConfig, Orchestrator orchestrator, - TesterClient testerClient) { + TesterClient testerClient, + Metric metric) { this(tenantRepository, hostProvisionerProvider.getHostProvisioner(), infraDeployerProvider.getInfraDeployer(), @@ -129,7 +135,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye new LogRetriever(), new FileDistributionStatus(), Clock.systemUTC(), - testerClient); + testerClient, + metric); } // For testing @@ -143,7 +150,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye new ConfigserverConfig(new ConfigserverConfig.Builder()), new LogRetriever(), clock, - new TesterClient()); + new TesterClient(), + new NullMetric()); } // For testing @@ -153,7 +161,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ConfigserverConfig configserverConfig, LogRetriever logRetriever, Clock clock, - TesterClient testerClient) { + TesterClient testerClient, + Metric metric) { this(tenantRepository, Optional.of(hostProvisioner), Optional.empty(), @@ -164,7 +173,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye logRetriever, new FileDistributionStatus(), clock, - testerClient); + testerClient, + metric); } private ApplicationRepository(TenantRepository tenantRepository, @@ -177,7 +187,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LogRetriever logRetriever, FileDistributionStatus fileDistributionStatus, Clock clock, - TesterClient testerClient) { + TesterClient testerClient, + Metric metric) { this.tenantRepository = tenantRepository; this.hostProvisioner = hostProvisioner; this.infraDeployer = infraDeployer; @@ -189,6 +200,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.fileDistributionStatus = fileDistributionStatus; this.clock = clock; this.testerClient = testerClient; + this.metric = metric; } // ---------------- Deploying ---------------------------------------------------------------- @@ -200,10 +212,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<ApplicationSet> currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, applicationId); Slime deployLog = createDeployLog(); DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); - ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); - logConfigChangeActions(actions, logger); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, actions, deployLog); + try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { + ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); + logConfigChangeActions(actions, logger); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + return new PrepareResult(sessionId, actions, deployLog); + } } public PrepareResult prepareAndActivate(Tenant tenant, long sessionId, PrepareParams prepareParams, @@ -361,15 +375,23 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // until the config server where the deployment happened picks it up and deletes // the local session long sessionId = activeSession.get(); - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - - if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - } else { - throw new InternalServerException("Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + RemoteSession remoteSession; + try { + remoteSession = getRemoteSession(tenant, sessionId); + Transaction deleteTransaction = remoteSession.createDeleteTransaction(); + deleteTransaction.commit(); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + + if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + } else { + deleteTransaction.rollbackOrLog(); + throw new InternalServerException(applicationId + " was not deleted (waited " + waitTime + "), session " + sessionId); + } + } catch (NotFoundException e) { + // For the case where waiting timed out in a previous attempt at deleting the application, continue and do the steps below + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); } NestedTransaction transaction = new NestedTransaction(); @@ -417,12 +439,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<String> fileReferencesInUse = new HashSet<>(); // Intentionally skip applications that we for some reason do not find - listApplications().stream() - .map(this::getOptionalApplication) - .map(Optional::get) - .forEach(application -> fileReferencesInUse.addAll(application.getModel().fileReferences().stream() - .map(FileReference::value) - .collect(Collectors.toSet()))); + // or that we fail to get file references for (they will be retried on the next run) + for (var application : listApplications()) { + try { + Optional<Application> app = getOptionalApplication(application); + if (app.isEmpty()) continue; + fileReferencesInUse.addAll(app.get().getModel().fileReferences().stream() + .map(FileReference::value) + .collect(Collectors.toSet())); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Getting file references in use for '" + application + "' failed", e); + } + } log.log(LogLevel.DEBUG, "File references in use : " + fileReferencesInUse); // Find those on disk that are not in use @@ -468,6 +496,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); @@ -832,4 +861,42 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye RegionName.from(configserverConfig.region())); } + /** Emits as a metric the time in millis spent while holding this timer, with deployment ID as dimensions. */ + public ActionTimer timerFor(ApplicationId id, String metricName) { + return new ActionTimer(metric, clock, id, configserverConfig.environment(), configserverConfig.region(), metricName); + } + + public static class ActionTimer implements AutoCloseable { + + private final Metric metric; + private final Clock clock; + private final ApplicationId id; + private final String environment; + private final String region; + private final String name; + private final Instant start; + + private ActionTimer(Metric metric, Clock clock, ApplicationId id, String environment, String region, String name) { + this.metric = metric; + this.clock = clock; + this.id = id; + this.environment = environment; + this.region = region; + this.name = name; + this.start = clock.instant(); + } + + @Override + public void close() { + metric.set(name, + Duration.between(start, clock.instant()).toMillis(), + metric.createContext(Map.of("tenant", id.tenant().value(), + "application", id.application().value(), + "instance", id.instance().value(), + "environment", environment, + "region", region))); + } + + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 9e81d3c0525..89d7c349d6b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -6,11 +6,14 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; @@ -22,6 +25,7 @@ import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.Optional; import java.util.logging.Logger; @@ -98,19 +102,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment { @Override public void prepare() { if (prepared) return; - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - - session.prepare(logger, - new PrepareParams.Builder().applicationId(session.getApplicationId()) - .timeoutBudget(timeoutBudget) - .ignoreValidationErrors( ! validate) - .vespaVersion(version.toString()) - .isBootstrap(isBootstrap) - .build(), - Optional.empty(), - tenant.getPath(), - clock.instant()); - this.prepared = true; + try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.prepareMillis")) { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + + session.prepare(logger, + new PrepareParams.Builder().applicationId(session.getApplicationId()) + .timeoutBudget(timeoutBudget) + .ignoreValidationErrors(!validate) + .vespaVersion(version.toString()) + .isBootstrap(isBootstrap) + .build(), + Optional.empty(), + tenant.getPath(), + clock.instant()); + this.prepared = true; + } } /** Activates this. If it is not already prepared, this will call prepare first. */ @@ -119,28 +125,32 @@ public class Deployment implements com.yahoo.config.provision.Deployment { if ( ! prepared) prepare(); - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - - ApplicationId applicationId = session.getApplicationId(); - try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { - validateSessionStatus(session); - NestedTransaction transaction = new NestedTransaction(); - transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure)); - hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); - transaction.commit(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new InternalServerException("Error activating application", e); - } + try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.activateMillis")) { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - session.waitUntilActivated(timeoutBudget); + ApplicationId applicationId = session.getApplicationId(); + try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { + validateSessionStatus(session); + NestedTransaction transaction = new NestedTransaction(); + transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure)); + hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); + transaction.commit(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + throw new InternalServerException("Error activating application", e); + } + + session.waitUntilActivated(timeoutBudget); - log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + - " activated successfully using " + - (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + - ". Config generation " + session.getMetaData().getGeneration() + - ". File references used: " + applicationRepository.getFileReferences(applicationId)); + log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + + " activated successfully using " + + (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + + ". Config generation " + session.getMetaData().getGeneration() + + ". File references used: " + applicationRepository.getFileReferences(applicationId)); + } } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 52d47a9398b..8b2c3e2cb0a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -130,7 +131,7 @@ public class ModelContextImpl implements ModelContext { private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useAdaptiveDispatch; - private final Optional<TlsSecrets> tlsSecrets; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final boolean useBucketSpaceMetric; @@ -146,7 +147,7 @@ public class ModelContextImpl implements ModelContext { boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, - Optional<TlsSecrets> tlsSecrets) { + Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -160,7 +161,7 @@ public class ModelContextImpl implements ModelContext { this.isFirstTimeDeployment = isFirstTimeDeployment; this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.tlsSecrets = tlsSecrets; + this.endpointCertificateSecrets = endpointCertificateSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource) @@ -208,7 +209,10 @@ public class ModelContextImpl implements ModelContext { public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override - public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); } + + @Override + public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index bc6419f230f..a2fc2bfd6a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -27,8 +27,9 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; @@ -135,7 +136,10 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating flagSource, - new TlsSecretsKeys(curator, TenantRepository.getTenantPath(tenant), secretStore).readTlsSecretsKeyFromZookeeper(applicationId)); + new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) + .readEndpointCertificateMetadata(applicationId) + .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets)); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index ab3e0e863ce..1a41c1efd7a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -11,6 +12,7 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import java.time.Clock; import java.time.Duration; @@ -32,6 +34,7 @@ public final class PrepareParams { static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; + static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -42,10 +45,12 @@ public final class PrepareParams { private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, - List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName) { + List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, + Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -55,6 +60,7 @@ public final class PrepareParams { this.vespaVersion = vespaVersion; this.containerEndpoints = containerEndpoints; this.tlsSecretsKeyName = tlsSecretsKeyName; + this.endpointCertificateMetadata = endpointCertificateMetadata; } public static class Builder { @@ -68,6 +74,7 @@ public final class PrepareParams { private Optional<Version> vespaVersion = Optional.empty(); private List<ContainerEndpoint> containerEndpoints = List.of(); private Optional<String> tlsSecretsKeyName = Optional.empty(); + private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); public Builder() { } @@ -128,9 +135,16 @@ public final class PrepareParams { return this; } + public Builder endpointCertificateMetadata(String serialized) { + if(serialized == null) return this; + Slime slime = SlimeUtils.jsonToSlime(serialized); + endpointCertificateMetadata = Optional.of(EndpointCertificateMetadataSerializer.fromSlime(slime.get())); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName); + verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata); } } @@ -144,6 +158,7 @@ public final class PrepareParams { .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) + .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) .build(); } @@ -200,4 +215,8 @@ public final class PrepareParams { public Optional<String> tlsSecretsKeyName() { return tlsSecretsKeyName; } + + public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { + return endpointCertificateMetadata; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 171eab35507..0115876ded9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -12,8 +12,9 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; @@ -33,7 +34,9 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import org.xml.sax.SAXException; @@ -113,7 +116,7 @@ public class SessionPreparer { preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { preparation.writeStateZK(); - preparation.writeTlsZK(); + preparation.writeEndpointCertificateMetadataZK(); preparation.writeContainerEndpointsZK(); preparation.distribute(); } @@ -142,8 +145,10 @@ public class SessionPreparer { final ContainerEndpointsCache containerEndpoints; final Set<ContainerEndpoint> endpointsSet; final ModelContext.Properties properties; - private final TlsSecretsKeys tlsSecretsKeys; - private final Optional<TlsSecrets> tlsSecrets; + private final EndpointCertificateMetadataStore endpointCertificateMetadataStore; + private final EndpointCertificateRetriever endpointCertificateRetriever; + private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -162,8 +167,16 @@ public class SessionPreparer { this.applicationId = params.getApplicationId(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); - this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); - this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId); + this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); + this.endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); + + this.endpointCertificateMetadata = params.endpointCertificateMetadata() + .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString)); + + endpointCertificateSecrets = endpointCertificateMetadata + .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + this.endpointsSet = getEndpoints(params.containerEndpoints()); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), @@ -178,7 +191,7 @@ public class SessionPreparer { params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), - tlsSecrets); + endpointCertificateSecrets); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -233,9 +246,10 @@ public class SessionPreparer { checkTimeout("write state to zookeeper"); } - void writeTlsZK() { - tlsSecretsKeys.writeTlsSecretsKeyToZooKeeper(applicationId, params.tlsSecretsKeyName().orElse(null)); - checkTimeout("write tlsSecretsKey to zookeeper"); + void writeEndpointCertificateMetadataZK() { + endpointCertificateMetadata.ifPresent(metadata -> + endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, metadata)); + checkTimeout("write endpoint certificate metadata to zookeeper"); } void writeContainerEndpointsZK() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java new file mode 100644 index 00000000000..6d092aaa18b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java @@ -0,0 +1,55 @@ +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +/** + * (de)serializes endpoint certificate metadata + * + * @author andreer + */ +public class EndpointCertificateMetadataSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private final static String keyNameField = "keyName"; + private final static String certNameField = "certName"; + private final static String versionField = "version"; + + public static void toSlime(EndpointCertificateMetadata metadata, Cursor object) { + object.setString(keyNameField, metadata.keyName()); + object.setString(certNameField, metadata.certName()); + object.setLong(versionField, metadata.version()); + } + + public static EndpointCertificateMetadata fromSlime(Inspector inspector) { + switch (inspector.type()) { + case STRING: // TODO: Remove once all are transmitted and stored as JSON + return new EndpointCertificateMetadata( + inspector.asString() + "-key", + inspector.asString() + "-cert", + 0 + ); + case OBJECT: + return new EndpointCertificateMetadata( + inspector.field(keyNameField).asString(), + inspector.field(certNameField).asString(), + Math.toIntExact(inspector.field(versionField).asLong()) + ); + + default: + throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); + } + } + + public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) { + return fromSlime(new Slime().setString(tlsSecretsKeys)); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java new file mode 100644 index 00000000000..6500449e557 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Optional; + +/** + * Stores the endpoint certificate metadata for an application. + * This metadata is then used to retrieve the actual secrets from {@link EndpointCertificateRetriever}. + * + * @author andreer + */ +public class EndpointCertificateMetadataStore { + + private final Path path; + private final Curator curator; + + public EndpointCertificateMetadataStore(Curator curator, Path tenantPath) { + this.curator = curator; + this.path = tenantPath.append("tlsSecretsKeys/"); + } + + /** Reads the endpoint certificate metadata from ZooKeeper, if it exists */ + public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId application) { + try { + Optional<byte[]> data = curator.getData(endpointCertificateMetadataPathOf(application)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + Slime slime = SlimeUtils.jsonToSlime(data.get()); + EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get()); + return Optional.of(endpointCertificateMetadata); + } catch (Exception e) { + throw new RuntimeException("Error reading TLS secret key of " + application, e); + } + } + + /** Writes the endpoint certificate metadata to ZooKeeper */ + public void writeEndpointCertificateMetadata(ApplicationId application, EndpointCertificateMetadata endpointCertificateMetadata) { + try { + Slime slime = new Slime(); + EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject()); + curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime)); + } catch (Exception e) { + throw new RuntimeException("Could not write TLS secret key of " + application, e); + } + } + + /** Returns a transaction which deletes these tls secrets key if they exist */ + public CuratorTransaction delete(ApplicationId application) { + if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator); + return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator); + } + + /** Returns the path storing the tls secrets key for an application */ + private Path endpointCertificateMetadataPathOf(ApplicationId application) { + return path.append(application.serializedForm()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java new file mode 100644 index 00000000000..5f40e5e1411 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; + +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.cert.X509Certificate; +import java.util.Optional; + +/** + * Used to retrieve actual endpoint certificate/key from secret store. + * + * @author andreer + */ +public class EndpointCertificateRetriever { + + private final SecretStore secretStore; + + public EndpointCertificateRetriever(SecretStore secretStore) { + this.secretStore = secretStore; + } + + public Optional<EndpointCertificateSecrets> readEndpointCertificateSecrets(EndpointCertificateMetadata metadata) { + return Optional.of(readFromSecretStore(metadata)); + } + + private EndpointCertificateSecrets readFromSecretStore(EndpointCertificateMetadata endpointCertificateMetadata) { + try { + String cert = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version()); + String key = secretStore.getSecret(endpointCertificateMetadata.keyName(), endpointCertificateMetadata.version()); + + verifyKeyMatchesCertificate(endpointCertificateMetadata, cert, key); + + return new EndpointCertificateSecrets(cert, key); + } catch (RuntimeException e) { + // Assume not ready yet + return EndpointCertificateSecrets.MISSING; + } + } + + private void verifyKeyMatchesCertificate(EndpointCertificateMetadata endpointCertificateMetadata, String cert, String key) { + X509Certificate x509Certificate = X509CertificateUtils.fromPem(cert); + + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(key); + PublicKey publicKey = x509Certificate.getPublicKey(); + + if(!X509CertificateUtils.privateKeyMatchesPublicKey(privateKey, publicKey)) { + throw new IllegalArgumentException("Failed to retrieve endpoint secrets: Certificate and key data do not match for " + endpointCertificateMetadata); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java deleted file mode 100644 index da6fc490da9..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.model.api.TlsSecrets; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.path.Path; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.transaction.CuratorOperations; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; - -import java.util.Optional; - -/** - * TLS Secret keys for applications (used to retrieve actual certificate/key from secret store). Persisted in ZooKeeper. - * - * @author andreer - */ -public class TlsSecretsKeys { - - private final Path path; - private final SecretStore secretStore; - private final Curator curator; - - public TlsSecretsKeys(Curator curator, Path tenantPath, SecretStore secretStore) { - this.curator = curator; - this.path = tenantPath.append("tlsSecretsKeys/"); - this.secretStore = secretStore; - } - - public Optional<TlsSecrets> readTlsSecretsKeyFromZookeeper(ApplicationId application) { - try { - Optional<byte[]> data = curator.getData(tlsSecretsKeyOf(application)); - if (data.isEmpty() || data.get().length == 0) return Optional.empty(); - - Slime slime = SlimeUtils.jsonToSlime(data.get()); - final var inspector = slime.get(); - - switch (inspector.type()) { - case STRING: // TODO: Remove once all are stored as JSON - return readFromSecretStore(Optional.ofNullable(inspector.asString())); - case OBJECT: - var tlsSecretsInfo = new TlsSecretsMetadata(); - tlsSecretsInfo.certName = inspector.field("certName").asString(); - tlsSecretsInfo.keyName = inspector.field("keyName").asString(); - tlsSecretsInfo.version = Math.toIntExact(inspector.field("version").asLong()); - return Optional.of(readFromSecretStore(tlsSecretsInfo)); - default: - throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!"); - } - } catch (Exception e) { - throw new RuntimeException("Error reading TLS secret key of " + application, e); - } - } - - public void writeTlsSecretsKeyToZooKeeper(ApplicationId application, String tlsSecretsKey) { - if (tlsSecretsKey == null) return; - writeTlsSecretsAsString(application, tlsSecretsKey); - } - - private void writeTlsSecretsAsString(ApplicationId application, String tlsSecretsKey) { - try { - Slime slime = new Slime(); - slime.setString(tlsSecretsKey); - curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - void writeTlsSecretsMetadata(ApplicationId application, TlsSecretsMetadata tlsSecretsMetadata) { - try { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString(TlsSecretsMetadata.certNameField, tlsSecretsMetadata.certName); - cursor.setString(TlsSecretsMetadata.keyNameField, tlsSecretsMetadata.keyName); - cursor.setLong(TlsSecretsMetadata.versionField, tlsSecretsMetadata.version); - curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime)); - } catch (Exception e) { - throw new RuntimeException("Could not write TLS secret key of " + application, e); - } - } - - public Optional<TlsSecrets> getTlsSecrets(Optional<String> secretKeyname, ApplicationId applicationId) { - if (secretKeyname == null || secretKeyname.isEmpty()) { - return readTlsSecretsKeyFromZookeeper(applicationId); - } - return readFromSecretStore(secretKeyname); - } - - private Optional<TlsSecrets> readFromSecretStore(Optional<String> secretKeyname) { - if (secretKeyname.isEmpty()) return Optional.empty(); - try { - String cert = secretStore.getSecret(secretKeyname.get() + "-cert"); - String key = secretStore.getSecret(secretKeyname.get() + "-key"); - return Optional.of(new TlsSecrets(cert, key)); - } catch (RuntimeException e) { - // Assume not ready yet - return Optional.of(TlsSecrets.MISSING); - } - } - - private TlsSecrets readFromSecretStore(TlsSecretsMetadata tlsSecretsMetadata) { - try { - String cert = secretStore.getSecret(tlsSecretsMetadata.certName, tlsSecretsMetadata.version); - String key = secretStore.getSecret(tlsSecretsMetadata.keyName, tlsSecretsMetadata.version); - return new TlsSecrets(cert, key); - } catch (RuntimeException e) { - // Assume not ready yet - return TlsSecrets.MISSING; - } - } - - /** Returns a transaction which deletes these tls secrets key if they exist */ - public CuratorTransaction delete(ApplicationId application) { - if (!curator.exists(tlsSecretsKeyOf(application))) return CuratorTransaction.empty(curator); - return CuratorTransaction.from(CuratorOperations.delete(tlsSecretsKeyOf(application).getAbsolute()), curator); - } - - /** Returns the path storing the tls secrets key for an application */ - private Path tlsSecretsKeyOf(ApplicationId application) { - return path.append(application.serializedForm()); - } - - static class TlsSecretsMetadata { - final static String keyNameField = "keyName"; - final static String certNameField = "certName"; - final static String versionField = "version"; - String keyName; - String certName; - int version; - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index fb745bbb76b..a963252d7ca 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -12,7 +12,9 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.application.OrchestratorMock; @@ -40,6 +42,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -266,13 +270,22 @@ public class ApplicationRepositoryTest { } { + PrepareResult prepareResult = deployApp(testApp); try { - deployApp(testApp); applicationRepository.delete(applicationId(), Duration.ZERO); fail("Should have gotten an exception"); } catch (InternalServerException e) { - assertEquals("Session 5 was not deleted (waited PT0S)", e.getMessage()); + assertEquals("test1.testapp was not deleted (waited PT0S), session " + prepareResult.sessionId(), e.getMessage()); } + + // No active session or remote session (deleted in step above), but an exception was thrown above + // A new delete should cleanup and be successful + LocalSession activeSession = applicationRepository.getActiveSession(applicationId()); + assertNull(activeSession); + Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); + assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId())); + + assertTrue(applicationRepository.delete(applicationId())); } } @@ -317,6 +330,29 @@ public class ApplicationRepositoryTest { assertEquals(0, applicationRepository.deleteExpiredRemoteSessions(Duration.ofSeconds(0))); } + @Test + public void testMetrics() { + MockMetric actual = new MockMetric(); + applicationRepository = new ApplicationRepository(tenantRepository, + provisioner, + orchestrator, + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new MockLogRetriever(), + new ManualClock(), + new MockTesterClient(), + actual); + deployApp(testAppLogServerWithContainer); + Map<String, ?> context = Map.of("tenant", "test1", + "application", "testapp", + "instance", "default", + "environment", "prod", + "region", "default"); + MockMetric expected = new MockMetric(); + expected.set("deployment.prepareMillis", 0L, expected.createContext(context)); + expected.set("deployment.activateMillis", 0L, expected.createContext(context)); + assertEquals(expected.values, actual.values); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, @@ -324,7 +360,8 @@ public class ApplicationRepositoryTest { new ConfigserverConfig(new ConfigserverConfig.Builder()), new MockLogRetriever(), clock, - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); } private PrepareResult prepareAndActivateApp(File application) throws IOException { @@ -360,4 +397,39 @@ public class ApplicationRepositoryTest { return applicationRepository.getMetadataFromSession(tenant, sessionId); } + + /** Stores all added or set values for each metric and context. */ + static class MockMetric implements Metric { + + final Map<String, Map<Map<String, ?>, Number>> values = new HashMap<>(); + + @Override + public void set(String key, Number val, Metric.Context ctx) { + values.putIfAbsent(key, new HashMap<>()); + values.get(key).put(((Context) ctx).point, val); + } + + @Override + public void add(String key, Number val, Metric.Context ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public Context createContext(Map<String, ?> properties) { + return new Context(properties); + } + + + private static class Context implements Metric.Context { + + private final Map<String, ?> point; + + public Context(Map<String, ?> point) { + this.point = Map.copyOf(point); + } + + } + + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java index 8a77b53875e..12f48778144 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java @@ -7,22 +7,26 @@ import java.util.HashMap; import java.util.Map; public class MockSecretStore implements SecretStore { - Map<String, String> secrets = new HashMap<>(); + Map<String, Map<Integer, String>> secrets = new HashMap<>(); @Override public String getSecret(String key) { if(secrets.containsKey(key)) - return secrets.get(key); + return secrets.get(key).get(0); throw new RuntimeException("Key not found: " + key); } @Override public String getSecret(String key, int version) { - return getSecret(key); + return secrets.get(key).get(version); + } + + public void put(String key, int version, String value) { + secrets.computeIfAbsent(key, k -> new HashMap<>()).put(version, value); } public void put(String key, String value) { - secrets.put(key, value); + put(key, 0, value); } public void remove(String key) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 9a15d6528ee..32b704dd551 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -27,6 +27,7 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockTesterClient; @@ -137,7 +138,8 @@ public class DeployTester { configserverConfig, new LogRetriever(), clock, - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); } public Tenant tenant() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 9e8d483cd86..438ccc5d783 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MockLogRetriever; @@ -79,7 +80,8 @@ public class ApplicationHandlerTest { new ConfigserverConfig(new ConfigserverConfig.Builder()), new MockLogRetriever(), Clock.systemUTC(), - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); @@ -177,7 +179,8 @@ public class ApplicationHandlerTest { mockHttpProxy, new ConfigserverConfig(new ConfigserverConfig.Builder()), new OrchestratorMock(), - new MockTesterClient()); + new MockTesterClient(), + new NullMetric()); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index a099db5ebe8..40115170b69 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ContainerEndpoint; -import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -22,6 +22,11 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockReloadHandler; @@ -37,7 +42,8 @@ import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; -import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; +import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -46,9 +52,14 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import javax.security.auth.x500.X500Principal; import java.io.File; import java.io.IOException; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -73,6 +84,9 @@ public class SessionPreparerTest { private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = new Version(1, 2, 3); private static final Version version321 = new Version(3, 2, 1); + private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private MockCurator curator; @@ -231,15 +245,37 @@ public class SessionPreparerTest { var tlskey = "vespa.tlskeys.tenant1--app1"; var applicationId = applicationId("test"); var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); - secretStore.put(tlskey+"-cert", "CERT"); - secretStore.put(tlskey+"-key", "KEY"); + + secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); + prepare(new File("src/test/resources/deploy/hosted-app"), params); // Read from zk and verify cert and key are available - Optional<TlsSecrets> tlsSecrets = new TlsSecretsKeys(curator, tenantPath, secretStore).readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) + .readEndpointCertificateMetadata(applicationId) + .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void require_that_endpoint_certificate_metadata_is_written() throws IOException { + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build(); + secretStore.put("vespa.tlskeys.tenant1--app1-cert", 7, X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", 7, KeyUtils.toPem(keyPair.getPrivate())); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + // Read from zk and verify cert and key are available + Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath) + .readEndpointCertificateMetadata(applicationId) + .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p)); + + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); } @Test(expected = CertificateNotReadyException.class) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java new file mode 100644 index 00000000000..d71eab25ce3 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java @@ -0,0 +1,90 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.EndpointCertificateMetadata; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.config.server.MockSecretStore; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class EndpointCertificateMetadataStoreTest { + + private static final Path tenantPath = Path.createRoot(); + private static final Path endpointCertificateMetadataPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); + private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), + ApplicationName.from("test"), InstanceName.defaultName()); + + private MockCurator curator; + private MockSecretStore secretStore = new MockSecretStore(); + private EndpointCertificateMetadataStore endpointCertificateMetadataStore; + private EndpointCertificateRetriever endpointCertificateRetriever; + private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"), + Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build(); + + @Before + public void setUp() { + curator = new MockCurator(); + endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); + endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore); + + secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate)); + secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate())); + } + + @Test + public void reads_string_format() { + curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes()); + + // Read from zk and verify cert and key are available + var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + assertTrue(endpointCertificateSecrets.isPresent()); + assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void reads_object_format() { + curator.set(endpointCertificateMetadataPath, + "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" + .getBytes()); + + // Read from zk and verify cert and key are available + var secrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId) + .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); + assertTrue(secrets.isPresent()); + assertTrue(secrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY")); + assertTrue(secrets.get().certificate().startsWith("-----BEGIN CERTIFICATE")); + } + + @Test + public void can_write_object_format() { + var endpointCertificateMetadata = new EndpointCertificateMetadata("key-name", "cert-name", 1); + + endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, endpointCertificateMetadata); + + assertEquals("{\"keyName\":\"key-name\",\"certName\":\"cert-name\",\"version\":1}", + new String(curator.getData(endpointCertificateMetadataPath).orElseThrow())); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java deleted file mode 100644 index c71c7b8e040..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.MockSecretStore; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TlsSecretsKeysTest { - - private static final Path tenantPath = Path.createRoot(); - private static final Path tlsSecretsKeysPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default"); - private static final String tlskey = "vespa.tlskeys.tenant1--app1"; - private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(), - ApplicationName.from("test"), InstanceName.defaultName()); - - private MockCurator curator; - private MockSecretStore secretStore = new MockSecretStore(); - private TlsSecretsKeys tlsSecretsKeys; - - @Before - public void setUp() { - curator = new MockCurator(); - tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); - secretStore.put(tlskey + "-cert", "CERT"); - secretStore.put(tlskey + "-key", "KEY"); - } - - @Test - public void reads_string_format() { - curator.set(tlsSecretsKeysPath, ('"' + tlskey + '"').getBytes()); - - // Read from zk and verify cert and key are available - var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); - } - - @Test - public void reads_object_format() { - curator.set(tlsSecretsKeysPath, - "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}" - .getBytes()); - - // Read from zk and verify cert and key are available - var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId); - assertTrue(tlsSecrets.isPresent()); - assertEquals("KEY", tlsSecrets.get().key()); - assertEquals("CERT", tlsSecrets.get().certificate()); - } - - @Test - public void can_write_object_format() { - var tlsSecretsMetadata = new TlsSecretsKeys.TlsSecretsMetadata(); - tlsSecretsMetadata.certName = "cert-name"; - tlsSecretsMetadata.keyName = "key-name"; - tlsSecretsMetadata.version = 1; - - tlsSecretsKeys.writeTlsSecretsMetadata(applicationId, tlsSecretsMetadata); - - assertEquals("{\"certName\":\"cert-name\",\"keyName\":\"key-name\",\"version\":1}", - new String(curator.getData(tlsSecretsKeysPath).get())); - } -} diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 5dc9a863970..91c0e4cc6bf 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -854,4 +854,4 @@ ], "fields": [] } -} +}
\ No newline at end of file diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 30cdbbfe531..a621545446f 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -446,7 +446,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.25.v20191220</jetty.version> + <jetty.version>9.4.26.v20200117</jetty.version> <lz4.version>1.3.0</lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> diff --git a/container-search/src/main/java/com/yahoo/prelude/Ping.java b/container-search/src/main/java/com/yahoo/prelude/Ping.java index dd14e150d95..1d5d4c92827 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Ping.java +++ b/container-search/src/main/java/com/yahoo/prelude/Ping.java @@ -4,7 +4,7 @@ package com.yahoo.prelude; /** * A ping, typically to ask whether backend is alive. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class Ping { diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index a60fba9a4f7..1e5513f1274 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -4,82 +4,102 @@ package com.yahoo.prelude; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.statistics.ElapsedTime; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * An answer from Ping. * - * @author Steinar Knutsen + * @author bratseth */ public class Pong { - private String pingInfo=""; - private final List<ErrorMessage> errors = new ArrayList<>(1); - private ElapsedTime elapsed = new ElapsedTime(); + private final ElapsedTime elapsed = new ElapsedTime(); private final Optional<Long> activeDocuments; + private final boolean isBlockingWrites; + private final Optional<ErrorMessage> error; public Pong() { - this.activeDocuments = Optional.empty(); + this(Optional.empty(), false, Optional.empty()); } public Pong(ErrorMessage error) { - errors.add(error); - this.activeDocuments = Optional.empty(); + this(Optional.empty(), false, Optional.of(error)); } public Pong(long activeDocuments) { - this.activeDocuments = Optional.of(activeDocuments); + this(Optional.of(activeDocuments), false, Optional.empty()); } - public void addError(ErrorMessage error) { - errors.add(error); + public Pong(long activeDocuments, boolean isBlockingWrites) { + this(Optional.of(activeDocuments), isBlockingWrites, Optional.empty()); } - public ErrorMessage getError(int i) { - return errors.get(i); + private Pong(Optional<Long> activeDocuments, boolean isBlockingWrites, Optional<ErrorMessage> error) { + this.activeDocuments = activeDocuments; + this.isBlockingWrites = isBlockingWrites; + this.error = error; } - /** Returns the number of active documents in the backend responding in this Pong, if available */ - public Optional<Long> activeDocuments() { - return activeDocuments; + /** + * @deprecated do not use. Additional errors are ignored. + */ + @Deprecated + public void addError(ErrorMessage error) { } + + /** + * @deprecated use error() instead + */ + @Deprecated + public ErrorMessage getError(int i) { + if (i > 1) throw new IllegalArgumentException("No error at position " + i); + if (i == 0 && error.isEmpty()) throw new IllegalArgumentException("No error at position " + i); + return error.get(); } - /** Returns the number of nodes which responded to this Pong, if available */ + public Optional<ErrorMessage> error() { return error; } + + /** Returns the number of active documents in the backend responding in this Pong, if available */ + public Optional<Long> activeDocuments() { return activeDocuments; } + + /** Returns true if the pinged node is currently blocking write operations due to being full */ + public boolean isBlockingWrites() { return isBlockingWrites; } + + /** + * Returns Optional.empty() + * + * @return empty + * @deprecated do not use. There is always one pong per node. + */ + @Deprecated public Optional<Integer> activeNodes() { return Optional.empty(); } + /** + * Returns a list containing 0 or 1 errors + * + * @deprecated use error() instead + */ + @Deprecated public List<ErrorMessage> getErrors() { - return Collections.unmodifiableList(errors); + return error.stream().collect(Collectors.toList()); } /** Returns whether there is an error or not */ - public boolean badResponse() { - return ! errors.isEmpty(); - } + public boolean badResponse() { return error.isPresent(); } - public ElapsedTime getElapsedTime() { - return elapsed; - } + public ElapsedTime getElapsedTime() { return elapsed; } /** Returns a string which included the ping info (if any) and any errors added to this */ @Override public String toString() { - StringBuilder m = new StringBuilder("Result of pinging"); - if (pingInfo.length() > 0) { - m.append(" using "); - m.append(pingInfo); - } - if (errors.size() > 0) - m.append(" "); - for (int i = 0; i < errors.size(); i++) { - m.append(errors.get(i).toString()); - if ( i <errors.size()-1) - m.append(", "); - } + StringBuilder m = new StringBuilder("Ping result"); + activeDocuments.ifPresent(docCount -> m.append(" active docs: ").append(docCount)); + if (isBlockingWrites) + m.append(" blocking writes: true"); + error.ifPresent(e -> m.append(" error: ").append(error)); return m.toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java index 153606e6d99..69131b6c690 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java @@ -8,7 +8,7 @@ import java.nio.ByteBuffer; /** - * <p>A set of terms which must be near each other to match.</p> + * A set of terms which must be near each other to match. * * @author bratseth * @author havardpe @@ -18,7 +18,7 @@ public class NearItem extends CompositeItem { protected int distance; /** The default distance used if none is specified: 2 */ - public static final int defaultDistance=2; + public static final int defaultDistance = 2; /** Creates a NEAR item with distance 2 */ public NearItem() { @@ -26,8 +26,7 @@ public class NearItem extends CompositeItem { } /** - * Creates a <i>near</i> item with a limit to the distance - * between the words. + * Creates a <i>near</i> item with a limit to the distance between the words. * * @param distance the number of word position which may separate * the words for this near item to match @@ -47,14 +46,17 @@ public class NearItem extends CompositeItem { return distance; } + @Override public ItemType getItemType() { return ItemType.NEAR; } + @Override public String getName() { return "NEAR"; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); IntegerCompressor.putCompressedPositiveNumber(distance, buffer); @@ -67,6 +69,7 @@ public class NearItem extends CompositeItem { } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); @@ -75,6 +78,7 @@ public class NearItem extends CompositeItem { buffer.append(" "); } + @Override public int hashCode() { return super.hashCode() + 23* distance; } @@ -83,10 +87,11 @@ public class NearItem extends CompositeItem { * Returns whether this item is of the same class and * contains the same state as the given item */ + @Override public boolean equals(Object object) { if (!super.equals(object)) return false; NearItem other = (NearItem) object; // Ensured by superclass - if (this.distance !=other.distance) return false; + if (this.distance != other.distance) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java index 84e93d5de8f..88982195af6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java @@ -25,10 +25,12 @@ public class ONearItem extends NearItem { super(distance); } + @Override public ItemType getItemType() { return ItemType.ONEAR; } + @Override public String getName() { return "ONEAR"; } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 06cbf9a9706..20f56c86f7b 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -96,7 +96,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod future.cancel(true); if (pong.badResponse()) { - monitor.failed(p, pong.getError(0)); + monitor.failed(p, pong.error().get()); log(LogLevel.FINE, "Failed ping - ", pong); } else { monitor.responded(p); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index e54e2187818..bc0a38617ee 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -13,6 +13,7 @@ import java.util.Optional; * @author bratseth */ interface Client { + /** Creates a connection to a particular node in this */ NodeConnection createConnection(String hostname, int port); @@ -21,6 +22,7 @@ interface Client { } class ResponseOrError<T> { + final Optional<T> response; final Optional<String> error; @@ -93,6 +95,7 @@ interface Client { } class ProtobufResponse { + private final byte compression; private final int uncompressedSize; private final byte[] compressedPayload; @@ -114,6 +117,7 @@ interface Client { public byte[] compressedPayload() { return compressedPayload; } + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index f2e22ba86dc..e0f1dc5e675 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -18,6 +18,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; public class RpcPing implements Callable<Pong> { + private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; @@ -57,8 +58,7 @@ public class RpcPing implements Callable<Pong> { var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); - connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), - timeoutSeconds); + connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), timeoutSeconds); } private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { @@ -67,12 +67,13 @@ public class RpcPing implements Callable<Pong> { var reply = SearchProtocol.MonitorReply.parseFrom(responseBytes); if (reply.getDistributionKey() != node.key()) { - return new Pong(ErrorMessage.createBackendCommunicationError( - "Expected pong from node id " + node.key() + ", response is from id " + reply.getDistributionKey())); + return new Pong(ErrorMessage.createBackendCommunicationError("Expected pong from node id " + node.key() + + ", response is from id " + reply.getDistributionKey())); } else if (!reply.getOnline()) { return new Pong(ErrorMessage.createBackendCommunicationError("Node id " + node.key() + " reports being offline")); } else { - return new Pong(reply.getActiveDocs()); + return new Pong(reply.getActiveDocs(), reply.getIsBlockingWrites()); } } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index eff6ae26816..5f211c37917 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -274,7 +274,7 @@ public class SearchCluster implements NodeManager<Node> { futurePong.cancel(true); if (pong.badResponse()) { - clusterMonitor.failed(node, pong.getError(0)); + clusterMonitor.failed(node, pong.error().get()); } else { if (pong.activeDocuments().isPresent()) { node.setActiveDocuments(pong.activeDocuments().get()); diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java index 2992d8ab896..8dcc25e4b3b 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java @@ -56,11 +56,10 @@ public class ClusterSearcherTestCase { @Override public Pong ping(Ping ping, Execution execution) { - Pong pong = new Pong(); - if (isBlocking()) { - pong.addError(ErrorMessage.createTimeout("Dummy timeout")); - } - return new Pong(); + if (isBlocking()) + return new Pong(ErrorMessage.createTimeout("Dummy timeout")); + else + return new Pong(); } public boolean isBlocking() { diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index c90d2774bd1..a824edd1996 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -182,10 +182,10 @@ public class ClusteredConnectionTestCase { @Override public Pong ping(Ping ping,Connection connection) { - Pong pong = new Pong(); - if (connection.getResponse() == null) - pong.addError(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); - return pong; + if (connection.getResponse() != null) + return new Pong(); + else + return new Pong(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 74e4ac9d404..a009f002954 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.InputStream; @@ -87,4 +88,8 @@ public interface ConfigServer { /** List all flag data for the given zone */ List<FlagData> listFlagData(ZoneId zone); + /** Gets status for tester application */ + // TODO: Remove default implementation when implemented in internal repo + default TesterCloud.Status getTesterStatus(DeploymentId deployment) { return TesterCloud.Status.SUCCESS; } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 7f61632ea4e..bbe533db9b5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.component.Version; +import java.net.URI; import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -13,6 +14,7 @@ import java.util.OptionalLong; * * @author bratseth * @author mpolden + * @author jonmv */ public class ApplicationVersion implements Comparable<ApplicationVersion> { @@ -39,24 +41,22 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { public ApplicationVersion(Optional<SourceRevision> source, OptionalLong buildNumber, Optional<String> authorEmail, Optional<Version> compileVersion, Optional<Instant> buildTime, Optional<String> sourceUrl, Optional<String> commit) { - Objects.requireNonNull(source, "source cannot be null"); - Objects.requireNonNull(buildNumber, "buildNumber cannot be null"); - Objects.requireNonNull(authorEmail, "author cannot be null"); - if (source.isPresent() != buildNumber.isPresent()) { - throw new IllegalArgumentException("both buildNumber and source must be set together"); - } - if (compileVersion.isPresent() != buildTime.isPresent()) { - throw new IllegalArgumentException("both compileVersion and buildTime must be set together"); - } - if (buildNumber.isPresent() && buildNumber.getAsLong() <= 0) { - throw new IllegalArgumentException("buildNumber must be > 0"); - } - if (authorEmail.isPresent() && ! authorEmail.get().matches("[^@]+@[^@]+")) { + if (buildNumber.isEmpty() && ( source.isPresent() || authorEmail.isPresent() || compileVersion.isPresent() + || buildTime.isPresent() || sourceUrl.isPresent() || commit.isPresent())) + throw new IllegalArgumentException("Build number must be present if any other attribute is"); + + if (buildNumber.isPresent() && buildNumber.getAsLong() <= 0) + throw new IllegalArgumentException("Build number must be > 0"); + + if (commit.isPresent() && commit.get().length() > 128) + throw new IllegalArgumentException("Commit may not be longer than 128 characters"); + + if (authorEmail.isPresent() && ! authorEmail.get().matches("[^@]+@[^@]+")) throw new IllegalArgumentException("Invalid author email '" + authorEmail.get() + "'."); - } - if (compileVersion.isPresent() && compileVersion.get().equals(Version.emptyVersion)) { + + if (compileVersion.isPresent() && compileVersion.get().equals(Version.emptyVersion)) throw new IllegalArgumentException("The empty version is not a legal compile version."); - } + this.source = source; this.buildNumber = buildNumber; this.authorEmail = authorEmail; @@ -86,10 +86,10 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { } /** Creates an version from a completed build, an author email, and build meta data. */ - public static ApplicationVersion from(SourceRevision source, long buildNumber, String authorEmail, Version compileVersion, - Instant buildTime, Optional<String> sourceUrl, Optional<String> commit) { - return new ApplicationVersion(Optional.of(source), OptionalLong.of(buildNumber), Optional.of(authorEmail), - Optional.of(compileVersion), Optional.of(buildTime), sourceUrl, commit); + public static ApplicationVersion from(Optional<SourceRevision> source, long buildNumber, Optional<String> authorEmail, + Optional<Version> compileVersion, Optional<Instant> buildTime, + Optional<String> sourceUrl, Optional<String> commit) { + return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail, compileVersion, buildTime, sourceUrl, commit); } /** Returns an unique identifier for this version or "unknown" if version is not known */ @@ -146,15 +146,15 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof ApplicationVersion)) return false; + if ( ! (o instanceof ApplicationVersion)) return false; ApplicationVersion that = (ApplicationVersion) o; - return Objects.equals(source, that.source) && - Objects.equals(buildNumber, that.buildNumber); + return Objects.equals(buildNumber, that.buildNumber) + && Objects.equals(commit(), that.commit()); } @Override public int hashCode() { - return Objects.hash(source, buildNumber); + return Objects.hash(buildNumber, commit()); } @Override @@ -173,8 +173,8 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { @Override public int compareTo(ApplicationVersion o) { - if ( ! buildNumber().isPresent() || ! o.buildNumber().isPresent()) - return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Application package hash sorts first + if (buildNumber().isEmpty() || o.buildNumber().isEmpty()) + return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Unknown version sorts first return Long.compare(buildNumber().getAsLong(), o.buildNumber().getAsLong()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index ef97ef27a72..a395d61e933 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import java.net.URI; @@ -22,6 +23,10 @@ public interface TesterCloud { /** Returns the current status of the tester. */ Status getStatus(URI testerUrl); + /** Returns the current status of the tester. */ + // TODO: Remove default implementation when implementors have been updated + default Status getStatus(DeploymentId deploymentId) { return Status.FAILURE; } + /** Returns whether the container is ready to serve. */ boolean ready(URI endpointUrl); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java index 0237f3d34f4..1d23cd52d23 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java @@ -19,12 +19,15 @@ public class ApplicationSummary { private final ApplicationId application; private final Optional<Instant> lastQueried; private final Optional<Instant> lastWritten; + private final Optional<Instant> lastBuilt; private final Map<ZoneId, Metric> metrics; - public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten, Map<ZoneId, Metric> metrics) { + public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten, + Optional<Instant> lastBuilt, Map<ZoneId, Metric> metrics) { this.application = Objects.requireNonNull(application); this.lastQueried = Objects.requireNonNull(lastQueried); this.lastWritten = Objects.requireNonNull(lastWritten); + this.lastBuilt = Objects.requireNonNull(lastBuilt); this.metrics = Map.copyOf(Objects.requireNonNull(metrics)); } @@ -40,6 +43,10 @@ public class ApplicationSummary { return lastWritten; } + public Optional<Instant> lastBuilt() { + return lastBuilt; + } + public Map<ZoneId, Metric> metrics() { return metrics; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index ce4d44eadce..e08141c422b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -1,13 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.google.common.collect.ImmutableList; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import java.net.URI; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -34,9 +33,10 @@ public class MockTesterCloud implements TesterCloud { } @Override - public Status getStatus(URI testerUrl) { - return status; - } + public Status getStatus(URI testerUrl) { return status; } + + @Override + public Status getStatus(DeploymentId deploymentId) { return status; } @Override public boolean ready(URI testerUrl) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index dfc9574fcd7..82120f13b75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; @@ -59,7 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; @@ -686,9 +686,9 @@ public class ApplicationController { catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + id, e); } - return routingPolicies.get(id).stream() + return routingPolicies.get(id).values().stream() .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 4f6fe2ac2db..d3e21f0d399 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; @@ -70,6 +71,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; private final MavenRepository mavenRepository; + private final Metric metric; /** * Creates a controller @@ -77,22 +79,15 @@ public class Controller extends AbstractComponent implements ApplicationIdSource * @param curator the curator instance storing the persistent state of the controller. */ @Inject - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - AccessControl accessControl, - FlagSource flagSource, - MavenRepository mavenRepository, - ServiceRegistry serviceRegistry) { - this(curator, rotationsConfig, - accessControl, - com.yahoo.net.HostName::getLocalhost, flagSource, - mavenRepository, serviceRegistry); + public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, + MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric) { + this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, + mavenRepository, serviceRegistry, metric); } - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - AccessControl accessControl, - Supplier<String> hostnameSupplier, - FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry) { + public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, + Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, + ServiceRegistry serviceRegistry, Metric metric) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -101,7 +96,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); - + this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); metrics = new ConfigServerMetrics(serviceRegistry.configServer()); nameServiceForwarder = new NameServiceForwarder(curator); @@ -265,6 +260,10 @@ public class Controller extends AbstractComponent implements ApplicationIdSource return auditLogger; } + public Metric metric() { + return metric; + } + private Set<CloudName> clouds() { return zoneRegistry.zones().all().zones().stream() .map(ZoneApi::getCloudName) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index de3f76d50cd..5e83d0a71a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; @@ -82,7 +83,9 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which currently have failing jobs on the given version */ public InstanceList failingOn(Version version) { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().lastCompleted().on(version).isEmpty()); + return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing() + .not().withStatus(outOfCapacity) + .lastCompleted().on(version).isEmpty()); } /** Returns the subset of instances which are not pinned to a certain Vespa version. */ @@ -92,7 +95,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are not currently failing any jobs. */ public InstanceList failing() { - return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().not().withStatus(RunStatus.outOfCapacity).isEmpty()); + return matching(id -> ! statuses.get(id).instanceJobs().get(id).failing().not().withStatus(outOfCapacity).isEmpty()); } /** Returns the subset of instances which are currently failing an upgrade. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java deleted file mode 100644 index 80a62d94f2e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; - -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * Represents the DNS routing policy for a load balancer. A routing policy is uniquely identified by its owner, cluster - * and zone. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicy { - - private final ApplicationId owner; - private final ClusterSpec.Id cluster; - private final ZoneId zone; - private final HostName canonicalName; - private final Optional<String> dnsZone; - private final Set<EndpointId> endpoints; - private final boolean active; - - /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName, - Optional<String> dnsZone, Set<EndpointId> endpoints, boolean active) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); - this.zone = Objects.requireNonNull(zone, "zone must be non-null"); - this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); - this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); - this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null")); - this.active = active; - } - - /** The application owning this */ - public ApplicationId owner() { - return owner; - } - - /** The zone this applies to */ - public ZoneId zone() { - return zone; - } - - /** The cluster this applies to */ - public ClusterSpec.Id cluster() { - return cluster; - } - - /** The canonical name for this (rhs of a CNAME or ALIAS record) */ - public HostName canonicalName() { - return canonicalName; - } - - /** DNS zone for this, if any */ - public Optional<String> dnsZone() { - return dnsZone; - } - - /** The endpoints of this policy */ - public Set<EndpointId> endpoints() { - return endpoints; - } - - /** Returns whether this is active (the underlying load balancer is in an active state) */ - public boolean active() { - return active; - } - - /** Returns the endpoint of this */ - public Endpoint endpointIn(SystemName system) { - return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system); - } - - /** Returns rotation endpoints of this */ - public EndpointList rotationEndpointsIn(SystemName system) { - return EndpointList.of(endpoints.stream().map(endpointId -> endpointOf(owner, endpointId, system))); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RoutingPolicy that = (RoutingPolicy) o; - return owner.equals(that.owner) && - cluster.equals(that.cluster) && - zone.equals(that.zone); - } - - @Override - public int hashCode() { - return Objects.hash(owner, cluster, zone); - } - - @Override - public String toString() { - return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, endpoints, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(), - zone.value()); - } - - /** Returns the endpoint of given rotation */ - public static Endpoint endpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { - return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 51841396d6a..35d478fb332 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -196,6 +196,13 @@ public class DeploymentTrigger { instance -> instance.withJobPause(jobType, OptionalLong.of(until.toEpochMilli()))))); } + /** Resumes a previously paused job, letting it be triggered normally. */ + public void resumeJob(ApplicationId id, JobType jobType) { + applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> + applications().store(application.with(id.instance(), + instance -> instance.withJobPause(jobType, OptionalLong.empty())))); + } + /** Triggers a change of this application, unless it already has a change. */ public void triggerChange(ApplicationId instanceId, Change change) { applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index bd61d85fbc0..9d09394a571 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -17,6 +17,9 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -469,7 +472,7 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> endTests(RunId id, DualLogger logger) { - if ( ! deployment(id.application(), id.type()).isPresent()) { + if (deployment(id.application(), id.type()).isEmpty()) { logger.log(INFO, "Deployment expired before tests could complete."); return Optional.of(aborted); } @@ -485,15 +488,22 @@ public class InternalStepRunner implements StepRunner { } } - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if ( ! testerEndpoint.isPresent()) { - logger.log("Endpoints for tester not found -- trying again later."); - return Optional.empty(); - } - controller.jobController().updateTestLog(id); - TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); + BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource()); + ZoneId zoneId = id.type().zone(controller.system()); + TesterCloud.Status testStatus; + if (useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value()) { + testStatus = controller.serviceRegistry().configServer().getTesterStatus(new DeploymentId(id.application(), zoneId)); + } else { + Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); + if (testerEndpoint.isEmpty()) { + logger.log("Endpoints for tester not found -- trying again later."); + return Optional.empty(); + } + testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); + } + switch (testStatus) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index e998e4c8ef9..c8cfc8ac286 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,9 +1,10 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -79,6 +80,7 @@ public class JobController { private final BufferedLogStore logs; private final TesterCloud cloud; private final Badges badges; + private final JobMetrics metric; private AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); @@ -88,6 +90,7 @@ public class JobController { this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); this.cloud = controller.serviceRegistry().testerCloud(); this.badges = new Badges(controller.zoneRegistry().badgeUrl()); + this.metric = new JobMetrics(controller.metric(), controller.system()); } public TesterCloud cloud() { return cloud; } @@ -360,6 +363,7 @@ public class JobController { } }); logs.flush(id); + metric.jobFinished(run.id().job(), finishedRun.status()); return finishedRun; }); } @@ -372,7 +376,7 @@ public class JobController { /** * Accepts and stores a new application package and test jar pair under a generated application version key. */ - public ApplicationVersion submit(TenantAndApplicationId id, SourceRevision revision, String authorEmail, + public ApplicationVersion submit(TenantAndApplicationId id, Optional<SourceRevision> revision, Optional<String> authorEmail, Optional<String> sourceUrl, Optional<String> commit, long projectId, ApplicationPackage applicationPackage, byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); @@ -380,14 +384,11 @@ public class JobController { long run = 1 + application.get().latestVersion() .map(latestVersion -> latestVersion.buildNumber().getAsLong()) .orElse(0L); - if (applicationPackage.compileVersion().isPresent() && applicationPackage.buildTime().isPresent()) - version.set(ApplicationVersion.from(revision, run, authorEmail, - applicationPackage.compileVersion().get(), - applicationPackage.buildTime().get(), - sourceUrl, - commit)); - else - version.set(ApplicationVersion.from(revision, run, authorEmail)); + version.set(ApplicationVersion.from(revision, run, authorEmail, + applicationPackage.compileVersion(), + applicationPackage.buildTime(), + sourceUrl, + commit)); controller.applications().applicationStore().put(id.tenant(), id.application(), @@ -419,6 +420,7 @@ public class JobController { RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant())); + metric.jobStarted(newId.job()); }); }); } @@ -529,7 +531,7 @@ public class JobController { DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); return controller.applications().getDeploymentEndpoints(testerId) .stream().findAny() - .or(() -> controller.applications().routingPolicies().get(testerId).stream() + .or(() -> controller.applications().routingPolicies().get(testerId).values().stream() .findAny() .map(policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java new file mode 100644 index 00000000000..a6ffb56492f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -0,0 +1,64 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; + +import java.util.Map; + +/** + * Records metrics related to deployment jobs. + * + * @author jonmv + */ +public class JobMetrics { + + public static final String start = "deployment.start"; + public static final String outOfCapacity = "deployment.outOfCapacity"; + public static final String deploymentFailure = "deployment.deploymentFailure"; + public static final String convergenceFailure = "deployment.convergenceFailure"; + public static final String testFailure = "deployment.testFailure"; + public static final String error = "deployment.error"; + public static final String abort = "deployment.abort"; + public static final String success = "deployment.success"; + + private final Metric metric; + private final SystemName system; + + public JobMetrics(Metric metric, SystemName system) { + this.metric = metric; + this.system = system; + } + + public void jobStarted(JobId id) { + metric.add(start, 1, metric.createContext(contextOf(id))); + } + + public void jobFinished(JobId id, RunStatus status) { + metric.add(valueOf(status), 1, metric.createContext(contextOf(id))); + } + + Map<String, String> contextOf(JobId id) { + return Map.of("tenant", id.application().tenant().value(), + "application", id.application().application().value(), + "instance", id.application().instance().value(), + "job", id.type().jobName(), + "environment", id.type().environment().value(), + "region", id.type().zone(system).region().value()); + } + + static String valueOf(RunStatus status) { + switch (status) { + case outOfCapacity: return outOfCapacity; + case deploymentFailed: return deploymentFailure; + case installationFailed: return convergenceFailure; + case testFailure: return testFailure; + case error: return error; + case aborted: return abort; + case success: return success; + default: throw new IllegalArgumentException("Unexpected run status '" + status + "'"); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 39faffcb869..c854c5b45bf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -84,8 +84,8 @@ public class ApplicationOwnershipConfirmer extends Maintainer { deploymentMetrics.queriesPerSecond(), deploymentMetrics.writesPerSecond())); } - // TODO jonmv: Default instance should really be replaced with something better. - return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), metrics); + return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(), + app.latestVersion().flatMap(version -> version.buildTime()), metrics); } /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java deleted file mode 100644 index ee38b2c9516..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Updates routing policies and their associated DNS records based on an deployment's load balancers. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicies { - - private static final Logger LOGGER = Logger.getLogger(RoutingPolicies.class.getName()); - - private final Controller controller; - private final CuratorDb db; - - public RoutingPolicies(Controller controller) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.db = controller.curator(); - try (var lock = db.lockRoutingPolicies()) { // Update serialized format - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - /** Read all known routing policies for given instance */ - public Set<RoutingPolicy> get(ApplicationId application) { - return db.readRoutingPolicies(application); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(DeploymentId deployment) { - return get(deployment.applicationId(), deployment.zoneId()); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(ApplicationId application, ZoneId zone) { - return db.readRoutingPolicies(application).stream() - .filter(policy -> policy.zone().equals(zone)) - .collect(Collectors.toUnmodifiableSet()); - } - - /** - * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if - * load balancers for given application have changed. - */ - public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; - var lbs = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer().getLoadBalancers(application, zone), - deploymentSpec); - try (var lock = db.lockRoutingPolicies()) { - removeObsoleteEndpointsFromDns(lbs, lock); - storePoliciesOf(lbs, lock); - removeObsoletePolicies(lbs, lock); - registerEndpointsInDns(lbs, lock); - } - } - - /** Create global endpoints for given route, if any */ - private void registerEndpointsInDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application)); - - // Create DNS record for each routing ID - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().endpointId(), - controller.system()); - Set<AliasTarget> targets = routeEntry.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } - } - - /** Store routing policies for given route. Returns the persisted policies. */ - private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashSet<>(get(loadBalancers.application)); - for (LoadBalancer loadBalancer : loadBalancers.list) { - var endpointIds = loadBalancers.endpointIdsOf(loadBalancer); - var policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer, endpointIds); - if (!policies.add(policy)) { - // Update existing policy - policies.remove(policy); - policies.add(policy); - } - } - db.writeRoutingPolicies(loadBalancers.application, policies); - } - - /** Create a policy for given load balancer and register a CNAME for it */ - private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer, - Set<EndpointId> endpointIds) { - var routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(), - loadBalancer.dnsZone(), endpointIds, isActive(loadBalancer)); - var name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName()); - var data = RecordData.fqdn(loadBalancer.hostname().value()); - controller.nameServiceForwarder().createCname(name, data, Priority.normal); - return routingPolicy; - } - - /** Remove obsolete policies for given route and their CNAME records */ - private void removeObsoletePolicies(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); - var removalCandidates = new HashSet<>(allPolicies); - var activeLoadBalancers = loadBalancers.list.stream() - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Remove active load balancers and irrelevant zones from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || - !policy.zone().equals(loadBalancers.zone)); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller.system()).dnsName(); - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - allPolicies.remove(policy); - } - db.writeRoutingPolicies(loadBalancers.application, allPolicies); - } - - /** Remove unreferenced global endpoints for given route from DNS */ - private void removeObsoleteEndpointsFromDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.application, loadBalancers.zone); - var removalCandidates = routingTableFrom(zonePolicies).keySet(); - var activeRoutingIds = routingIdsFrom(loadBalancers); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - var endpoint = RoutingPolicy.endpointOf(id.application(), id.endpointId(), controller.system()); - controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } - } - - /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { - Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : loadBalancers.list) { - for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { - routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Compute a routing table from given policies */ - private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) { - var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); - for (var policy : routingPolicies) { - for (var rotation : policy.endpoints()) { - var id = new RoutingId(policy.owner(), rotation); - routingTable.putIfAbsent(id, new ArrayList<>()); - routingTable.get(id).add(policy); - } - } - return routingTable; - } - - private static boolean isActive(LoadBalancer loadBalancer) { - switch (loadBalancer.state()) { - case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early - // as possible - case active: return true; - } - return false; - } - - /** Load balancers allocated to a deployment */ - private static class AllocatedLoadBalancers { - - private final ApplicationId application; - private final ZoneId zone; - private final List<LoadBalancer> list; - private final DeploymentSpec deploymentSpec; - - private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, - DeploymentSpec deploymentSpec) { - this.application = application; - this.zone = zone; - this.list = List.copyOf(loadBalancers); - this.deploymentSpec = deploymentSpec; - } - - /** Compute all endpoint IDs for given load balancer */ - private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { - if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints - return Set.of(); - } - var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); - if (instanceSpec.isEmpty()) { - return Set.of(); - } - return instanceSpec.get().endpoints().stream() - .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.regions().contains(zone.region())) - .map(com.yahoo.config.application.api.Endpoint::endpointId) - .map(EndpointId::of) - .collect(Collectors.toSet()); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 22894a084b6..1a2ffc69249 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -18,19 +18,21 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.apache.zookeeper.data.Stat; import java.io.IOException; import java.io.UncheckedIOException; @@ -80,6 +82,7 @@ public class CuratorDb { private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); + private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); private static final Path applicationCertificateRoot = root.append("applicationCertificates"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -93,6 +96,7 @@ public class CuratorDb { private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); + private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); @@ -485,19 +489,28 @@ public class CuratorDb { // -------------- Routing policies ---------------------------------------- - public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) { + public void writeRoutingPolicies(ApplicationId application, Map<RoutingPolicyId, RoutingPolicy> policies) { curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies))); } - public Map<ApplicationId, Set<RoutingPolicy>> readRoutingPolicies() { + public Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> readRoutingPolicies() { return curator.getChildren(routingPoliciesRoot).stream() .map(ApplicationId::fromSerializedForm) .collect(Collectors.toUnmodifiableMap(Function.identity(), this::readRoutingPolicies)); } - public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) { + public Map<RoutingPolicyId, RoutingPolicy> readRoutingPolicies(ApplicationId application) { return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime)) - .orElseGet(Collections::emptySet); + .orElseGet(Map::of); + } + + public void writeZoneRoutingPolicy(ZoneRoutingPolicy policy) { + curator.set(zoneRoutingPolicyPath(policy.zone()), asJson(zoneRoutingPolicySerializer.toSlime(policy))); + } + + public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) { + return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data)) + .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS)); } // -------------- Application web certificates ---------------------------- @@ -581,6 +594,8 @@ public class CuratorDb { return routingPoliciesRoot.append(application.serializedForm()); } + private static Path zoneRoutingPolicyPath(ZoneId zone) { return zoneRoutingPoliciesRoot.append(zone.value()); } + private static Path nameServiceQueuePath() { return root.append("nameServiceQueue"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 54a3ef7551a..2429c5ee8c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; @@ -6,13 +6,20 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; +import java.time.Instant; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Set; +import java.util.Map; /** * Serializer and deserializer for a {@link RoutingPolicy}. @@ -35,45 +42,64 @@ public class RoutingPolicySerializer { private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; private static final String rotationsField = "rotations"; - private static final String activeField = "active"; + private static final String loadBalancerActiveField = "active"; + private static final String globalRoutingField = "globalRouting"; + private static final String agentField = "agent"; + private static final String changedAtField = "changedAt"; + private static final String statusField = "status"; - public Slime toSlime(Set<RoutingPolicy> routingPolicies) { + public Slime toSlime(Map<RoutingPolicyId, RoutingPolicy> routingPolicies) { var slime = new Slime(); var root = slime.setObject(); var policyArray = root.setArray(routingPoliciesField); - routingPolicies.forEach(policy -> { + routingPolicies.values().forEach(policy -> { var policyObject = policyArray.addObject(); - policyObject.setString(clusterField, policy.cluster().value()); - policyObject.setString(zoneField, policy.zone().value()); + policyObject.setString(clusterField, policy.id().cluster().value()); + policyObject.setString(zoneField, policy.id().zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); var rotationArray = policyObject.setArray(rotationsField); policy.endpoints().forEach(endpointId -> { rotationArray.addString(endpointId.id()); }); - policyObject.setBool(activeField, policy.active()); + policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); + globalRoutingToSlime(policy.status().globalRouting(), policyObject.setObject(globalRoutingField)); }); return slime; } - public Set<RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { - var policies = new LinkedHashSet<RoutingPolicy>(); + public Map<RoutingPolicyId, RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { + var policies = new LinkedHashMap<RoutingPolicyId, RoutingPolicy>(); var root = slime.get(); var field = root.field(routingPoliciesField); field.traverse((ArrayTraverser) (i, inspect) -> { var endpointIds = new LinkedHashSet<EndpointId>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, endpointId) -> endpointIds.add(EndpointId.of(endpointId.asString()))); - var activeFieldInspector = inspect.field(activeField); - // TODO(mpolden): Remove field presence check after January 2020 - boolean active = !activeFieldInspector.valid() || activeFieldInspector.asBool(); - policies.add(new RoutingPolicy(owner, - ClusterSpec.Id.from(inspect.field(clusterField).asString()), - ZoneId.from(inspect.field(zoneField).asString()), - HostName.from(inspect.field(canonicalNameField).asString()), - Serializers.optionalString(inspect.field(dnsZoneField)), - endpointIds, active)); + var id = new RoutingPolicyId(owner, + ClusterSpec.Id.from(inspect.field(clusterField).asString()), + ZoneId.from(inspect.field(zoneField).asString())); + policies.put(id, new RoutingPolicy(id, + HostName.from(inspect.field(canonicalNameField).asString()), + Serializers.optionalString(inspect.field(dnsZoneField)), + endpointIds, + new Status(inspect.field(loadBalancerActiveField).asBool(), + globalRoutingFromSlime(inspect.field(globalRoutingField))))); }); - return Collections.unmodifiableSet(policies); + return Collections.unmodifiableMap(policies); + } + + public void globalRoutingToSlime(GlobalRouting globalRouting, Cursor object) { + object.setString(statusField, globalRouting.status().name()); + object.setString(agentField, globalRouting.agent().name()); + object.setLong(changedAtField, globalRouting.changedAt().toEpochMilli()); + } + + public GlobalRouting globalRoutingFromSlime(Inspector object) { + if (!object.valid()) return GlobalRouting.DEFAULT_STATUS; + var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); + var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); + var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); + return new GlobalRouting(status, agent, changedAt); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java new file mode 100644 index 00000000000..6688d16ad14 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; + +import java.util.Objects; + +/** + * Serializer for {@link ZoneRoutingPolicy}. + * + * @author mpolden + */ +public class ZoneRoutingPolicySerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String GLOBAL_ROUTING_FIELD = "globalRouting"; + + private final RoutingPolicySerializer routingPolicySerializer; + + public ZoneRoutingPolicySerializer(RoutingPolicySerializer routingPolicySerializer) { + this.routingPolicySerializer = Objects.requireNonNull(routingPolicySerializer, "routingPolicySerializer must be non-null"); + } + + public ZoneRoutingPolicy fromSlime(ZoneId zone, Slime slime) { + var root = slime.get(); + return new ZoneRoutingPolicy(zone, routingPolicySerializer.globalRoutingFromSlime(root.field(GLOBAL_ROUTING_FIELD))); + } + + public Slime toSlime(ZoneRoutingPolicy policy) { + var slime = new Slime(); + var root = slime.setObject(); + routingPolicySerializer.globalRoutingToSlime(policy.globalRouting(), root.setObject(GLOBAL_ROUTING_FIELD)); + return slime; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index e92c52a613f..81d9f08925f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; @@ -59,6 +59,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; +import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; @@ -68,7 +71,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; @@ -81,6 +83,7 @@ import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -297,6 +300,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return resume(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -691,6 +695,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse(type.jobName() + " for " + id + " paused for " + DeploymentTrigger.maxPause); } + private HttpResponse resume(ApplicationId id, JobType type) { + controller.applications().deploymentTrigger().resumeJob(id, type); + return new MessageResponse(type.jobName() + " for " + id + " resumed"); + } + private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); @@ -798,9 +807,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .forEach(globalEndpointUrls::add); // Per-cluster endpoints. These are backed by load balancers. - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); + var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); for (var policy : routingPolicies) { - policy.rotationEndpointsIn(controller.system()).asList().stream() + policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalEndpointUrls::add); @@ -923,10 +932,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); - for (RoutingPolicy policy : routingPolicies) { - if (!policy.active()) continue; - policy.rotationEndpointsIn(controller.system()).asList().stream() + var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); + for (var policy : routingPolicies) { + if (!policy.status().isActive()) continue; + policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalRotationsArray::addString); @@ -1037,11 +1046,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add endpoint(s) defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies().get(deploymentId)) { - if (!policy.active()) continue; + for (var policy : controller.applications().routingPolicies().get(deploymentId).values()) { + if (!policy.status().isActive()) continue; Cursor endpointObject = endpointArray.addObject(); Endpoint endpoint = policy.endpointIn(controller.system()); - endpointObject.setString("cluster", policy.cluster().value()); + endpointObject.setString("cluster", policy.id().cluster().value()); endpointObject.setBool("tls", endpoint.tls()); endpointObject.setString("url", endpoint.url().toString()); } @@ -1180,26 +1189,39 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new NotExistsException(instance + " has no deployment in " + zone); } + // The order here matters because setGlobalRotationStatus involves an external request that may fail. + // TODO(mpolden): Set only one of these when only one kind of global endpoints are supported per zone. + var deploymentId = new DeploymentId(instance.id(), zone); + setGlobalRotationStatus(deploymentId, inService, request); + setGlobalEndpointStatus(deploymentId, inService, request); + + return new MessageResponse(String.format("Successfully set %s in %s %s service", + instance.id().toShortString(), zone, inService ? "in" : "out of")); + } + + /** Set the global endpoint status for given deployment. This only applies to global endpoints backed by a cloud service */ + private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { + var roles = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class).roles(); + var isOperator = roles.stream().map(Role::definition).anyMatch(d -> d == RoleDefinition.hostedOperator); + var agent = isOperator ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant; + var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out; + controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent); + } + + /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ + private void setGlobalRotationStatus(DeploymentId deployment, boolean inService, HttpRequest request) { Inspector requestData = toSlime(request.getData()).get(); String reason = mandatory("reason", requestData).asString(); String agent = requireUserPrincipal(request).getName(); long timestamp = controller.clock().instant().getEpochSecond(); EndpointStatus.Status status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; EndpointStatus endpointStatus = new EndpointStatus(status, reason, agent, timestamp); - controller.applications().setGlobalRotationStatus(new DeploymentId(instance.id(), deployment.zone()), - endpointStatus); - return new MessageResponse(String.format("Successfully set %s in %s.%s %s service", - instance.id().toShortString(), - deployment.zone().environment().value(), - deployment.zone().region().value(), - inService ? "in" : "out of")); + controller.applications().setGlobalRotationStatus(deployment, endpointStatus); } private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { - DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region)); - Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); controller.applications().globalRotationStatus(deploymentId) @@ -1211,7 +1233,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { statusObject.setString("agent", status.getAgent() == null ? "" : status.getAgent()); statusObject.setLong("timestamp", status.getEpoch()); }); - return new SlimeJsonResponse(slime); } @@ -1988,11 +2009,20 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse submit(String tenant, String application, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); - SourceRevision sourceRevision = toSourceRevision(submitOptions); - String authorEmail = submitOptions.field("authorEmail").asString(); long projectId = Math.max(1, submitOptions.field("projectId").asLong()); - Optional<String> commit = submitOptions.field("commit").valid() ? Optional.of(submitOptions.field("commit").asString()) : Optional.empty(); - Optional<String> sourceUrl = submitOptions.field("sourceUrl").valid() ? Optional.of(submitOptions.field("sourceUrl").asString()) : Optional.empty(); + Optional<String> repository = optional("repository", submitOptions); + Optional<String> branch = optional("branch", submitOptions); + Optional<String> commit = optional("commit", submitOptions); + Optional<SourceRevision> sourceRevision = repository.isPresent() && branch.isPresent() && commit.isPresent() + ? Optional.of(new SourceRevision(repository.get(), branch.get(), commit.get())) + : Optional.empty(); + Optional<String> sourceUrl = optional("sourceUrl", submitOptions); + Optional<String> authorEmail = optional("authorEmail", submitOptions); + + sourceUrl.map(URI::create).ifPresent(url -> { + if (url.getHost() == null || url.getScheme() == null) + throw new IllegalArgumentException("Source URL must include scheme and host"); + }); ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), true); @@ -2052,5 +2082,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return "UNKNOWN"; } + private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) { + return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName)) + .filter(cls::isInstance) + .map(cls::cast) + .orElseThrow(() -> new IllegalArgumentException("Attribute '" + attributeName + "' was not set on request")); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 4bcf744ead5..8cc309c33aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -472,8 +472,8 @@ class JobControllerApiHandlerHelper { * @return Response with the new application version */ static HttpResponse submitResponse(JobController jobController, String tenant, String application, - SourceRevision sourceRevision, String authorEmail, Optional<String> sourceUrl, - Optional<String> commit, long projectId, + Optional<SourceRevision> sourceRevision, Optional<String> authorEmail, + Optional<String> sourceUrl, Optional<String> commit, long projectId, ApplicationPackage applicationPackage, byte[] testPackage) { ApplicationVersion version = jobController.submit(TenantAndApplicationId.from(tenant, application), sourceRevision, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java new file mode 100644 index 00000000000..1b2cf4a7896 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java @@ -0,0 +1,85 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.time.Instant; +import java.util.Objects; + +/** + * Represents the global routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. This contains the + * time global routing status was last changed and who changed it. + * + * This is immutable. + * + * @author mpolden + */ +public class GlobalRouting { + + public static final GlobalRouting DEFAULT_STATUS = new GlobalRouting(Status.in, Agent.system, Instant.EPOCH); + + private final Status status; + private final Agent agent; + private final Instant changedAt; + + /** DO NOT USE. Public for serialization purposes */ + public GlobalRouting(Status status, Agent agent, Instant changedAt) { + this.status = Objects.requireNonNull(status, "status must be non-null"); + this.agent = Objects.requireNonNull(agent, "agent must be non-null"); + this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + } + + /** The current status of this */ + public Status status() { + return status; + } + + /** The agent who last changed this */ + public Agent agent() { + return agent; + } + + /** The time this was last changed */ + public Instant changedAt() { + return changedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GlobalRouting that = (GlobalRouting) o; + return status == that.status && + agent == that.agent && + changedAt.equals(that.changedAt); + } + + @Override + public int hashCode() { + return Objects.hash(status, agent, changedAt); + } + + @Override + public String toString() { + return "status " + status + ", changed by " + agent + " @ " + changedAt; + } + + public static GlobalRouting status(Status status, Agent agent, Instant instant) { + return new GlobalRouting(status, agent, instant); + } + + // Used in serialization. Do not change. + public enum Status { + /** Status is determined by health checks **/ + in, + + /** Status is explicitly set to out */ + out, + } + + /** Agents that can change the state of global routing */ + public enum Agent { + operator, + tenant, + system, + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java index 7b0ec3d27ba..5543d0ea0b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java @@ -1,7 +1,8 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import java.util.Objects; @@ -42,4 +43,9 @@ public class RoutingId { return Objects.hash(application, endpointId); } + @Override + public String toString() { + return "routing id for " + endpointId + " of " + application; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java new file mode 100644 index 00000000000..c05152e7795 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -0,0 +1,288 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicies { + + private final Controller controller; + private final CuratorDb db; + + public RoutingPolicies(Controller controller) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.db = controller.curator(); + try (var lock = db.lockRoutingPolicies()) { // Update serialized format + for (var policy : db.readRoutingPolicies().entrySet()) { + db.writeRoutingPolicies(policy.getKey(), policy.getValue()); + } + } + } + + /** Read all known routing policies for given instance */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application) { + return db.readRoutingPolicies(application); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) { + return get(deployment.applicationId(), deployment.zoneId()); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) { + return db.readRoutingPolicies(application).entrySet() + .stream() + .filter(kv -> kv.getKey().zone().equals(zone)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if + * load balancers for given application have changed. + */ + public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { + if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; + var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(application, zone), + deploymentSpec); + var inactiveZones = inactiveZones(application, deploymentSpec); + try (var lock = db.lockRoutingPolicies()) { + removeGlobalDnsUnreferencedBy(loadBalancers, lock); + storePoliciesOf(loadBalancers, lock); + removePoliciesUnreferencedBy(loadBalancers, lock); + updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); + } + } + + /** Set the status of all global endpoints in given zone */ + public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) { + try (var lock = db.lockRoutingPolicies()) { + db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, + controller.clock().instant()))); + var allPolicies = db.readRoutingPolicies(); + for (var applicationPolicies : allPolicies.values()) { + updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock); + } + } + } + + /** Set the status of all global endpoints for given deployment */ + public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + try (var lock = db.lockRoutingPolicies()) { + var policies = get(deployment.applicationId()); + var newPolicies = new LinkedHashMap<>(policies); + for (var policy : policies.values()) { + if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone + var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent, + controller.clock().instant()))); + newPolicies.put(policy.id(), newPolicy); + } + db.writeRoutingPolicies(deployment.applicationId(), newPolicies); + updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); + } + } + + /** Update global DNS record for given policies */ + private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + // Create DNS record for each routing ID + var routingTable = routingTableFrom(routingPolicies); + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + var targets = new LinkedHashSet<AliasTarget>(); + var staleTargets = new LinkedHashSet<AliasTarget>(); + for (var policy : routeEntry.getValue()) { + if (policy.dnsZone().isEmpty()) continue; + var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); + var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); + // Remove target zone if global routing status is set out at: + // - zone level (ZoneRoutingPolicy) + // - deployment level (RoutingPolicy) + // - application package level (deployment.xml) + if (anyOut(zonePolicy.globalRouting(), policy.status().globalRouting()) || + inactiveZones.contains(policy.id().zone())) { + staleTargets.add(target); + } else { + targets.add(target); + } + } + if (!targets.isEmpty()) { + var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(), + routeEntry.getKey().endpointId(), controller.system()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + } + staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal)); + } + } + + /** Store routing policies for given load balancers */ + private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = new LinkedHashMap<>(get(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone); + var existingPolicy = policies.get(policyId); + var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancers.endpointIdsOf(loadBalancer), + new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); + // Preserve global routing status for existing policy + if (existingPolicy != null) { + newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().globalRouting())); + } + updateZoneDnsOf(newPolicy); + policies.put(newPolicy.id(), newPolicy); + } + db.writeRoutingPolicies(loadBalancers.application, policies); + } + + /** Update zone DNS record for given policy */ + private void updateZoneDnsOf(RoutingPolicy policy) { + var name = RecordName.from(policy.endpointIn(controller.system()).dnsName()); + var data = RecordData.fqdn(policy.canonicalName().value()); + controller.nameServiceForwarder().createCname(name, data, Priority.normal); + } + + /** Remove policies and zone DNS records unreferenced by given load balancers */ + private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = get(loadBalancers.application); + var newPolicies = new LinkedHashMap<>(policies); + var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); + for (var policy : policies.values()) { + // Leave active load balancers and irrelevant zones alone + if (activeLoadBalancers.contains(policy.canonicalName()) || + !policy.id().zone().equals(loadBalancers.zone)) continue; + + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + newPolicies.remove(policy.id()); + } + db.writeRoutingPolicies(loadBalancers.application, newPolicies); + } + + /** Remove unreferenced global endpoints from DNS */ + private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values(); + var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); + var activeRoutingIds = routingIdsFrom(loadBalancers); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + } + } + + /** Compute routing IDs from given load balancers */ + private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { + Set<RoutingId> routingIds = new LinkedHashSet<>(); + for (var loadBalancer : loadBalancers.list) { + for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { + routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); + } + } + return Collections.unmodifiableSet(routingIds); + } + + /** Compute a routing table from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) { + var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); + for (var policy : routingPolicies) { + for (var endpoint : policy.endpoints()) { + var id = new RoutingId(policy.id().owner(), endpoint); + routingTable.putIfAbsent(id, new ArrayList<>()); + routingTable.get(id).add(policy); + } + } + return Collections.unmodifiableMap(routingTable); + } + + private static boolean anyOut(GlobalRouting... globalRouting) { + return Arrays.stream(globalRouting) + .map(GlobalRouting::status) + .anyMatch(status -> status == GlobalRouting.Status.out); + } + + private static boolean isActive(LoadBalancer loadBalancer) { + switch (loadBalancer.state()) { + case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early + // as possible + case active: return true; + } + return false; + } + + /** Load balancers allocated to a deployment */ + private static class AllocatedLoadBalancers { + + private final ApplicationId application; + private final ZoneId zone; + private final List<LoadBalancer> list; + private final DeploymentSpec deploymentSpec; + + private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, + DeploymentSpec deploymentSpec) { + this.application = application; + this.zone = zone; + this.list = List.copyOf(loadBalancers); + this.deploymentSpec = deploymentSpec; + } + + /** Compute all endpoint IDs for given load balancer */ + private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { + if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints + return Set.of(); + } + var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); + if (instanceSpec.isEmpty()) { + return Set.of(); + } + return instanceSpec.get().endpoints().stream() + .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) + .filter(endpoint -> endpoint.regions().contains(zone.region())) + .map(com.yahoo.config.application.api.Endpoint::endpointId) + .map(EndpointId::of) + .collect(Collectors.toSet()); + } + + } + + /** Returns zones where global routing is declared inactive for instance through deploymentSpec */ + private static Set<ZoneId> inactiveZones(ApplicationId instance, DeploymentSpec deploymentSpec) { + var instanceSpec = deploymentSpec.instance(instance.instance()); + if (instanceSpec.isEmpty()) return Set.of(); + return instanceSpec.get().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .filter(zone -> !zone.active()) + .map(zone -> ZoneId.from(zone.environment(), zone.region().get())) + .collect(Collectors.toUnmodifiableSet()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java new file mode 100644 index 00000000000..9ef2d519c05 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -0,0 +1,106 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.google.common.collect.ImmutableSortedSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; + +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Represents the DNS routing policy for a {@link com.yahoo.vespa.hosted.controller.application.Deployment}. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicy { + + private final RoutingPolicyId id; + private final HostName canonicalName; + private final Optional<String> dnsZone; + private final Set<EndpointId> endpoints; + private final Status status; + + /** DO NOT USE. Public for serialization purposes */ + public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, Set<EndpointId> endpoints, + Status status) { + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); + this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); + this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null")); + this.status = Objects.requireNonNull(status, "status must be non-null"); + } + + /** The ID of this */ + public RoutingPolicyId id() { + return id; + } + + /** The canonical name for the load balancer this applies to (rhs of a CNAME or ALIAS record) */ + public HostName canonicalName() { + return canonicalName; + } + + /** DNS zone for the load balancer this applies to, if any. Used when creating ALIAS records. */ + public Optional<String> dnsZone() { + return dnsZone; + } + + /** The endpoints of this policy */ + public Set<EndpointId> endpoints() { + return endpoints; + } + + /** Returns the status of this */ + public Status status() { + return status; + } + + /** Returns a copy of this with status set to given status */ + public RoutingPolicy with(Status status) { + return new RoutingPolicy(id, canonicalName, dnsZone, endpoints, status); + } + + /** Returns the endpoint of this */ + public Endpoint endpointIn(SystemName system) { + return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system); + } + + /** Returns global endpoints which this is a member of */ + public EndpointList globalEndpointsIn(SystemName system) { + return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system))); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingPolicy that = (RoutingPolicy) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return String.format("%s [endpoints: %s%s], %s owned by %s, in %s", canonicalName, endpoints, + dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(), + id.zone().value()); + } + + /** Creates a global endpoint for given application */ + public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { + return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java new file mode 100644 index 00000000000..06002e874f1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java @@ -0,0 +1,57 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Unique identifier for a {@link RoutingPolicy}. + * + * @author mpolden + */ +public class RoutingPolicyId { + + private final ApplicationId owner; + private final ClusterSpec.Id cluster; + private final ZoneId zone; + + public RoutingPolicyId(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone) { + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + } + + /** The application owning this */ + public ApplicationId owner() { + return owner; + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The cluster this applies to */ + public ClusterSpec.Id cluster() { + return cluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingPolicyId that = (RoutingPolicyId) o; + return owner.equals(that.owner) && + cluster.equals(that.cluster) && + zone.equals(that.zone); + } + + @Override + public int hashCode() { + return Objects.hash(owner, cluster, zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java new file mode 100644 index 00000000000..51e59c7cf4f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.util.Objects; + +/** + * Represents the status of a routing policy. + * + * This is immutable. + * + * @author mpolden + */ +public class Status { + + private final boolean active; + private final GlobalRouting globalRouting; + + /** DO NOT USE. Public for serialization purposes */ + public Status(boolean active, GlobalRouting globalRouting) { + this.active = active; + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** Returns whether this is considered active according to the load balancer status */ + public boolean isActive() { + return active; + } + + /** Return status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + /** Returns a copy of this with global routing changed */ + public Status with(GlobalRouting globalRouting) { + return new Status(active, globalRouting); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return active == status.active && + globalRouting.equals(status.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(active, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java new file mode 100644 index 00000000000..262cacd325e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Represents the DNS routing policy for a zone. This takes precedence over of an individual {@link RoutingPolicy}. + * + * This is immutable. + * + * @author mpolden + */ +public class ZoneRoutingPolicy { + + private final ZoneId zone; + private final GlobalRouting globalRouting; + + public ZoneRoutingPolicy(ZoneId zone, GlobalRouting globalRouting) { + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneRoutingPolicy that = (ZoneRoutingPolicy) o; + return zone.equals(that.zone) && + globalRouting.equals(that.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(zone, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index f722eb4f6bb..7c3c30738d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -1,17 +1,13 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import java.time.Instant; import java.time.ZoneOffset; -import java.util.stream.Collectors; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 84bdedba33c..c83463bc1ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -355,7 +356,6 @@ public final class ControllerTester { return application; } - public void deploy(ApplicationId id, ZoneId zone) { deploy(id, zone, new ApplicationPackage(new byte[0])); } @@ -392,7 +392,8 @@ public final class ControllerTester { () -> "test-controller", new InMemoryFlagSource(), new MockMavenRepository(), - serviceRegistry); + serviceRegistry, + new MetricsMock()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index e4b5e77b377..9b0706d184f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -101,13 +101,19 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder region(RegionName regionName) { - return region(regionName.value()); + return region(regionName, true); } public ApplicationPackageBuilder region(String regionName) { - environmentBody.append(" <region active='true'>"); - environmentBody.append(regionName); - environmentBody.append("</region>\n"); + return region(RegionName.from(regionName), true); + } + + public ApplicationPackageBuilder region(RegionName regionName, boolean active) { + environmentBody.append(" <region active='") + .append(active) + .append("'>") + .append(regionName.value()) + .append("</region>\n"); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index bd3dc5d4336..2792a59b523 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -1,10 +1,12 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -26,9 +28,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; @@ -38,6 +45,7 @@ import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -57,8 +65,8 @@ import static org.junit.Assert.assertTrue; * * References to this should be acquired through {@link DeploymentTester#newDeploymentContext}. * - * Tester code that is not specific to deployments should be added to either {@link ControllerTester} or - * {@link DeploymentTester} instead of this class. + * Tester code that is not specific to a single application's deployment context should be added to either + * {@link ControllerTester} or {@link DeploymentTester} instead of this class. * * @author mpolden * @author jonmv @@ -197,6 +205,28 @@ public class DeploymentContext { return this; } + /** Add a routing policy for this in given zone, with status set to active */ + public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) { + return addRoutingPolicy(instanceId, zone, active); + } + + /** Add a routing policy for tester instance of this in given zone, with status set to active */ + public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) { + return addRoutingPolicy(testerId.id(), zone, active); + } + + private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) { + var clusterId = "default" + (!active ? "-inactive" : ""); + var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone); + var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance)); + policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"), + Optional.empty(), + Set.of(EndpointId.of("c0")), + new Status(active, GlobalRouting.DEFAULT_STATUS))); + tester.controller().curator().writeRoutingPolicies(instance, policies); + return this; + } + /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { return submit(applicationPackage, defaultSourceRevision); @@ -208,7 +238,8 @@ public class DeploymentContext { .requireApplication(applicationId) .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. - lastSubmission = jobs.submit(applicationId, sourceRevision, "a@b", Optional.empty(), Optional.empty(), projectId, applicationPackage, new byte[0]); + lastSubmission = jobs.submit(applicationId, Optional.of(sourceRevision), Optional.of("a@b"), Optional.empty(), + Optional.empty(), projectId, applicationPackage, new byte[0]); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 51726035cb3..e052b967c31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -1,11 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; @@ -24,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Before; import org.junit.Test; @@ -43,20 +41,18 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.Future; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.publicCdApplicationPackage; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -208,19 +204,8 @@ public class InternalStepRunnerTest { tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - - tester.controller().curator().writeRoutingPolicies(app.instanceId(), Set.of(new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("default"), - JobType.systemTest.zone(system()), - HostName.from("host"), - Optional.empty(), - emptySet(), true))); - tester.controller().curator().writeRoutingPolicies(app.testerId().id(), Set.of(new RoutingPolicy(app.testerId().id(), - ClusterSpec.Id.from("default"), - JobType.systemTest.zone(system()), - HostName.from("host"), - Optional.empty(), - emptySet(), true))); + app.addRoutingPolicy(JobType.systemTest.zone(system()), true); + app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true); tester.runner().run();; assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index b37d3d340cb..fef8ab32d17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -269,6 +270,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return List.of(); } + @Override + public TesterCloud.Status getTesterStatus(DeploymentId deployment) { + return TesterCloud.Status.SUCCESS; + } + public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { this.loadBalancers.putIfAbsent(zone, new ArrayList<>()); this.loadBalancers.get(zone).addAll(loadBalancers); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index b2ec7ec5f26..9de0020ce4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -11,12 +11,14 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; +import com.yahoo.vespa.hosted.controller.deployment.JobMetrics; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; import com.yahoo.vespa.hosted.controller.deployment.Versions; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Test; import java.time.Duration; @@ -43,6 +45,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; @@ -59,6 +62,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -87,7 +91,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source().get(), "a@b", Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); try { @@ -119,7 +123,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source().get(), "a@b", Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); Supplier<Run> run = () -> jobs.last(id, systemTest).get(); jobs.start(id, systemTest, versions); @@ -227,7 +231,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source().get(), "a@b", Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); RunId runId = new RunId(id, systemTest, 1); jobs.start(id, systemTest, versions); @@ -265,7 +269,7 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId instanceId = appId.defaultInstance(); JobId jobId = new JobId(instanceId, systemTest); - jobs.submit(appId, versions.targetApplication().source().get(), "a@b", Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); assertFalse(jobs.lastSuccess(jobId).isPresent()); for (int i = 0; i < jobs.historyLength(); i++) { @@ -343,12 +347,49 @@ public class JobRunnerTest { TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); - jobs.submit(appId, versions.targetApplication().source().get(), "a@b", Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); jobs.start(id, systemTest, versions); tester.clock().advance(JobRunner.jobTimeout.plus(Duration.ofSeconds(1))); runner.run(); - assertTrue(jobs.last(id, systemTest).get().status() == aborted); + assertSame(aborted, jobs.last(id, systemTest).get().status()); + } + + @Test + public void jobMetrics() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), + inThreadExecutor(), mappedRunner(outcomes)); + + TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); + ApplicationId id = appId.defaultInstance(); + jobs.submit(appId, versions.targetApplication().source(), Optional.empty(), Optional.empty(), Optional.empty(), 2, applicationPackage, new byte[0]); + + for (RunStatus status : RunStatus.values()) { + if (status == success) continue; // Status not used for steps. + outcomes.put(deployTester, status); + jobs.start(id, systemTest, versions); + runner.run(); + jobs.finish(jobs.last(id, systemTest).get().id()); + } + + Map<String, String> context = Map.of("tenant", "tenant", + "application", "real", + "instance", "default", + "job", "system-test", + "environment", "test", + "region", "us-east-1"); + MetricsMock metric = ((MetricsMock) tester.controller().metric()); + assertEquals(RunStatus.values().length - 1, metric.getMetric(context::equals, JobMetrics.start).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.abort).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.error).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.success).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } public static ExecutorService inThreadExecutor() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 23355bd6033..c9ec5adc98c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -1,22 +1,26 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; import org.junit.Test; +import java.time.Instant; import java.util.Iterator; +import java.util.Map; import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author mortent @@ -29,41 +33,46 @@ public class RoutingPolicySerializerTest { public void serialization() { var owner = ApplicationId.defaultId(); var endpoints = Set.of(EndpointId.of("r1"), EndpointId.of("r2")); - var policies = ImmutableSet.of(new RoutingPolicy(owner, - ClusterSpec.Id.from("my-cluster1"), - ZoneId.from("prod", "us-north-1"), + var id1 = new RoutingPolicyId(owner, + ClusterSpec.Id.from("my-cluster1"), + ZoneId.from("prod", "us-north-1")); + var id2 = new RoutingPolicyId(owner, + ClusterSpec.Id.from("my-cluster2"), + ZoneId.from("prod", "us-north-2")); + var policies = ImmutableMap.of(id1, new RoutingPolicy(id1, HostName.from("long-and-ugly-name"), Optional.of("zone1"), - endpoints, true), - new RoutingPolicy(owner, - ClusterSpec.Id.from("my-cluster2"), - ZoneId.from("prod", "us-north-2"), + endpoints, new Status(true, GlobalRouting.DEFAULT_STATUS)), + id2, new RoutingPolicy(id2, HostName.from("long-and-ugly-name-2"), Optional.empty(), - endpoints, false)); + endpoints, new Status(false, + new GlobalRouting(GlobalRouting.Status.out, + GlobalRouting.Agent.tenant, + Instant.ofEpochSecond(123))))); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); - for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext();) { + for (Iterator<RoutingPolicy> it1 = policies.values().iterator(), it2 = serialized.values().iterator(); it1.hasNext();) { var expected = it1.next(); var actual = it2.next(); - assertEquals(expected.owner(), actual.owner()); - assertEquals(expected.cluster(), actual.cluster()); - assertEquals(expected.zone(), actual.zone()); + assertEquals(expected.id(), actual.id()); assertEquals(expected.canonicalName(), actual.canonicalName()); assertEquals(expected.dnsZone(), actual.dnsZone()); assertEquals(expected.endpoints(), actual.endpoints()); - assertEquals(expected.active(), actual.active()); + assertEquals(expected.status(), actual.status()); } } + // TODO(mpolden): Remove after January 2020 @Test public void legacy_serialization() { - var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\"," + - "\"canonicalName\":\"lb-0\"," + - "\"dnsZone\":\"dns-zone-id\",\"rotations\":[]}]}"; - var serialized = serializer.fromSlime(ApplicationId.defaultId(), SlimeUtils.jsonToSlime(json)); - assertTrue(serialized.iterator().next().active()); - + var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\",\"canonicalName\":\"lb-host\",\"dnsZone\":\"dnsZoneId\",\"rotations\":[\"default\"],\"active\":true}]}"; + var owner = ApplicationId.defaultId(); + var serialized = serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json)); + var id = new RoutingPolicyId(owner, ClusterSpec.Id.from("default"), ZoneId.from("prod", "us-north-1")); + var expected = Map.of(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.of("dnsZoneId"), + Set.of(EndpointId.defaultId()), new Status(true, GlobalRouting.DEFAULT_STATUS))); + assertEquals(expected, serialized); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java new file mode 100644 index 00000000000..6a089c5e1b0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java @@ -0,0 +1,29 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; +import org.junit.Test; + +import java.time.Instant; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ZoneRoutingPolicySerializerTest { + + @Test + public void serialization() { + var serializer = new ZoneRoutingPolicySerializer(new RoutingPolicySerializer()); + var zone = ZoneId.from("prod", "us-north-1"); + var policy = new ZoneRoutingPolicy(zone, + GlobalRouting.status(GlobalRouting.Status.out, GlobalRouting.Agent.operator, + Instant.ofEpochMilli(123))); + var serialized = serializer.fromSlime(zone, serializer.toSlime(policy)); + assertEquals(policy, serialized); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index dbf2b8be2d4..388c245118b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; @@ -11,7 +11,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -52,8 +51,6 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -66,6 +63,7 @@ import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -81,6 +79,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.time.YearMonth; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -523,6 +522,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"production-us-west-1 for tenant1.application1.instance1 paused for " + DeploymentTrigger.maxPause + "\"}"); + // DELETE a pause of a production job + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/pause", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"production-us-west-1 for tenant1.application1.instance1 resumed\"}"); + // POST a triggering to the same production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", POST) .userIdentity(USER_ID), @@ -761,16 +765,20 @@ public class ApplicationApiTest extends ControllerContainerTest { public void testRotationOverride() { // Setup createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() .instances("instance1") .globalServiceId("foo") - .region("us-west-1") - .region("us-east-3") + .region(westZone.region()) + .region(eastZone.region()) .build(); // Create tenant and deploy var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); - app.submit(applicationPackage).runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); + app.submit(applicationPackage).deploy(); + app.addRoutingPolicy(westZone, true); + app.addRoutingPolicy(eastZone, true); // Invalid application fails tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation", GET) @@ -779,16 +787,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // Invalid deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", 404); // Change status of non-existing deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation/override", PUT) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", 404); // GET global rotation status @@ -808,11 +816,23 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-put.json")); + // Status of routing policy is changed + assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + // DELETE global rotation override status tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", DELETE) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-delete.json")); + assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + + // SET global rotation override status by operator + addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", PUT) + .userIdentity(HOSTED_VESPA_OPERATOR) + .data("{\"reason\":\"unit-test\"}"), + new File("global-rotation-put.json")); + assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.operator); } @Test @@ -1429,18 +1449,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("us-west-1") .build(); app.submit(applicationPackage).deploy(); - Set<RoutingPolicy> policies = Set.of(new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("default"), - ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - HostName.from("lb-0-canonical-name"), - Optional.of("dns-zone-1"), Set.of(EndpointId.of("c0")), true), - // Inactive policy is not included - new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("deleted-cluster"), - ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - HostName.from("lb-1-canonical-name"), - Optional.of("dns-zone-1"), Set.of(), false)); - tester.controller().curator().writeRoutingPolicies(app.instanceId(), policies); + app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true); + app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) @@ -1599,6 +1609,16 @@ public class ApplicationApiTest extends ControllerContainerTest { "queue", Optional.empty())); } + private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + var changedAt = tester.controller().clock().instant(); + var westPolicies = tester.controller().applications().routingPolicies().get(deployment); + assertEquals(1, westPolicies.size()); + var westPolicy = westPolicies.values().iterator().next(); + assertEquals(status, westPolicy.status().globalRouting().status()); + assertEquals(agent, westPolicy.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().globalRouting().changedAt()); + } + private static class RequestBuilder implements Supplier<Request> { private final String path; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 1bb20296bd2..c0420c7b895 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,5 +1,5 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -24,7 +24,12 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -68,21 +73,9 @@ public class RoutingPoliciesTest { // Creates alias records context1.submit(applicationPackage).deploy(); - var endpoint1 = "r0.app1.tenant1.global.vespa.oath.cloud"; - var endpoint2 = "r1.app1.tenant1.global.vespa.oath.cloud"; - var endpoint3 = "r2.app1.tenant1.global.vespa.oath.cloud"; - - assertEquals(endpoint1 + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint1)); - assertEquals(endpoint2 + " points to c0 us-west-1", - List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint2)); - assertEquals(endpoint3 + " points to c1 in all regions", - List.of("lb-1--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-1--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint3)); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); assertEquals("Routing policy count is equal to cluster count", numberOfDeployments * clustersPerZone, tester.policiesOf(context1.instance().id()).size()); @@ -100,12 +93,10 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); context1.submit(applicationPackage2).deploy(); - // Endpoint is updated to contain cluster in new deployment - assertEquals(endpoint1 + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app1:default--prod.us-east-3/dns-zone-1/prod.us-east-3", - "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint1)); + // Endpoints are updated to contain cluster in new deployment + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); // Another application is deployed with a single cluster and global endpoint var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; @@ -116,10 +107,7 @@ public class RoutingPoliciesTest { .endpoint("r0", "c0") .build(); context2.submit(applicationPackage3).deploy(); - assertEquals(endpoint4 + " points to c0 in all regions", - List.of("lb-0--tenant1:app2:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app2:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint4)); + tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // All endpoints for app1 are removed ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder() @@ -129,10 +117,10 @@ public class RoutingPoliciesTest { .allow(ValidationId.globalEndpointChange) .build(); context1.submit(applicationPackage4).deploy(); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint1)); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint2)); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint3)); - Set<RoutingPolicy> policies = tester.policiesOf(context1.instanceId()); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); + var policies = tester.policiesOf(context1.instanceId()); assertEquals(clustersPerZone * numberOfDeployments, policies.size()); assertTrue("Rotation membership is removed from all policies", policies.stream().allMatch(policy -> policy.endpoints().isEmpty())); @@ -226,8 +214,8 @@ public class RoutingPoliciesTest { "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); - assertTrue("Removes stale routing policies " + context2.application(), tester.controllerTester().controller().applications().routingPolicies().get(context2.instanceId()).isEmpty()); - assertEquals("Keeps routing policies for " + context1.application(), 4, tester.controllerTester().controller().applications().routingPolicies().get(context1.instanceId()).size()); + assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty()); + assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size()); } @Test @@ -348,6 +336,145 @@ public class RoutingPoliciesTest { assertEquals("CNAME points to current load blancer", newHostname.value() + ".", tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); } + + @Test + public void set_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + // Global DNS record is created + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + // Global routing status is overridden in one zone + var changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + + // Inactive zone is removed from global DNS record + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Status details is stored in policy + var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Other zone remains in + var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent()); + assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); + + // Next deployment does not affect status + context.submit(applicationPackage).deploy(); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Deployment is set back in + tester.controllerTester().clock().advance(Duration.ofHours(1)); + changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Deployment is set out through a new deployment.xml + var applicationPackage2 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region(), false) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage2).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); + + // ... back in + var applicationPackage3 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage3).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + } + + @Test + public void set_zone_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant2", "app2", "default"); + var contexts = List.of(context1, context2); + + // Deploy applications + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) + .build(); + for (var context : contexts) { + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } + + // Set zone out + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + for (var context : contexts) { + var policies = tester.routingPolicies().get(context.instanceId()); + assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, + policies.values().stream() + .map(RoutingPolicy::status) + .map(Status::globalRouting) + .map(GlobalRouting::status) + .allMatch(status -> status == GlobalRouting.Status.in)); + } + var changedAt = tester.controllerTester().clock().instant(); + var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); + assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status()); + assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt()); + + // Setting status per deployment does not affect status as entire zone is out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + + // Set single deployment out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + + // Set zone back in. Deployment set explicitly out, remains out, the rest are in + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); @@ -372,6 +499,10 @@ public class RoutingPoliciesTest { this(new DeploymentTester()); } + public RoutingPolicies routingPolicies() { + return tester.controllerTester().controller().applications().routingPolicies(); + } + public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { return tester.newDeploymentContext(tenant, application, instance); } @@ -391,8 +522,8 @@ public class RoutingPoliciesTest { } } - private Set<RoutingPolicy> policiesOf(ApplicationId instance) { - return tester.controller().curator().readRoutingPolicies(instance); + private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { + return tester.controller().curator().readRoutingPolicies(instance).values(); } private Set<String> recordNames() { @@ -416,6 +547,21 @@ public class RoutingPoliciesTest { .collect(Collectors.toList()); } + private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { + var prefix = ""; + if (!endpointId.equals(EndpointId.defaultId())) { + prefix = endpointId.id() + "."; + } + var endpoint = prefix + application.application().value() + "." + application.tenant().value() + + ".global.vespa.oath.cloud"; + var zoneTargets = Arrays.stream(zone) + .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + + z.value() + "/dns-zone-1/" + z.value()) + .collect(Collectors.toSet()); + assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, + Set.copyOf(aliasDataOf(endpoint))); + } + } } diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp index 014884e44cd..0bdf51194fb 100644 --- a/document/src/tests/arrayfieldvaluetest.cpp +++ b/document/src/tests/arrayfieldvaluetest.cpp @@ -15,9 +15,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -53,40 +52,39 @@ TEST(ArrayFieldValueTest, testArray) EXPECT_EQ(IntFieldValue(3), (IntFieldValue&) value[2]); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream stream(value.serialize()); ArrayFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(stream, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By index - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.remove(1); EXPECT_TRUE(!value2.contains(IntFieldValue(2))); EXPECT_EQ(size_t(2), value2.size()); // By value - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // By value with multiple present - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.add(IntFieldValue(1)); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index d75a16ea1a6..7735ccdbb1d 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/document/datatype/annotationreferencedatatype.h> #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/repo/configbuilder.h> @@ -9,6 +8,9 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> @@ -28,10 +30,14 @@ using namespace fieldvalue; TEST(DocumentTest, testSizeOf) { + EXPECT_EQ(24u, sizeof(std::vector<char>)); + EXPECT_EQ(24u, sizeof(vespalib::alloc::Alloc)); + EXPECT_EQ(40u, sizeof(ByteBuffer)); + EXPECT_EQ(32u, sizeof(vespalib::GrowableByteBuffer)); EXPECT_EQ(88ul, sizeof(IdString)); EXPECT_EQ(104ul, sizeof(DocumentId)); - EXPECT_EQ(208ul, sizeof(Document)); - EXPECT_EQ(72ul, sizeof(StructFieldValue)); + EXPECT_EQ(200ul, sizeof(Document)); + EXPECT_EQ(64ul, sizeof(StructFieldValue)); EXPECT_EQ(24ul, sizeof(StructuredFieldValue)); EXPECT_EQ(64ul, sizeof(SerializableArray)); } @@ -63,7 +69,7 @@ TEST(DocumentTest, testFieldPath) class Handler : public fieldvalue::IteratorHandler { public: Handler(); - ~Handler(); + ~Handler() override; const std::string & getResult() const { return _result; } private: void onPrimitive(uint32_t, const Content&) override { @@ -386,13 +392,13 @@ TEST(DocumentTest, testSimpleUsage) EXPECT_EQ(1, value.getValue(intF)->getAsInt()); EXPECT_EQ(2, value.getValue(longF)->getAsInt()); - // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + // Serialize & equality + nbostream buffer; + value.serialize(buffer); Document value2(*repo.getDocumentType("test"), DocumentId("id::test:n=3:foo")); EXPECT_TRUE(value != value2); - value2.deserialize(repo, *buffer); + value2.deserialize(repo, buffer); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); EXPECT_EQ(DocumentId("id:ns:test::1"), value2.getId()); @@ -400,15 +406,15 @@ TEST(DocumentTest, testSimpleUsage) // Various ways of removing { // By value - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); @@ -561,29 +567,31 @@ TEST(DocumentTest, testReadSerializedFile) int fd = open(TEST_PATH("data/serializejava.dat").c_str(), O_RDONLY); size_t len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != (ssize_t)len) { + if (read(fd, buf.get(), len) != (ssize_t)len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); verifyJavaDocument(doc2); - EXPECT_EQ(len, buf2->getPos()); - EXPECT_TRUE(memcmp(buf2->getBuffer(), buf.getBuffer(), buf2->getPos()) == 0); + EXPECT_TRUE(buf2.empty()); + buf2.rp(0); + EXPECT_EQ(len, buf2.size()); + EXPECT_TRUE(memcmp(buf2.peek(), buf.get(), buf2.size()) == 0); doc2.setValue("stringfield", StringFieldValue("hei")); - std::unique_ptr<ByteBuffer> buf3 = doc2.serialize(); - EXPECT_TRUE(len != buf3->getPos()); + nbostream buf3 = doc2.serialize(); + EXPECT_TRUE(len != buf3.size()); } TEST(DocumentTest, testReadSerializedFileCompressed) @@ -595,14 +603,15 @@ TEST(DocumentTest, testReadSerializedFileCompressed) int fd = open(TEST_PATH("data/serializejava-compressed.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); } @@ -706,26 +715,25 @@ TEST(DocumentTest,testReadSerializedAllVersions) // you can copy this current to new test for new version) { //doc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0)); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_EQ(buf->getLength(), buf->getPos()); + nbostream buf = doc.serialize(); int fd = open(TEST_PATH("data/document-cpp-currentversion-uncompressed.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); } { CompressionConfig oldCfg(doc.getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_TRUE(buf->getPos() <= buf->getLength()); + nbostream buf = doc.serialize(); + EXPECT_TRUE(buf.size() <= buf.capacity()); int fd = open(TEST_PATH("data/document-cpp-currentversion-lz4-9.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(oldCfg); } @@ -745,14 +753,15 @@ TEST(DocumentTest,testReadSerializedAllVersions) } int fd = open(tests[i]._dataFile.c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { - throw vespalib::Exception("read failed"); - } + if (read(fd, buf.get(), len) != len) { + throw vespalib::Exception("read failed"); + } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); IntFieldValue intVal; EXPECT_TRUE(doc.getValue(doc.getField("intfield"), intVal)); @@ -805,27 +814,14 @@ TEST(DocumentTest,testReadSerializedAllVersions) EXPECT_EQ(199, wset.get(StringFieldValue("Weighted 1"))); // Check that serialization doesn't cause any problems. - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); } } size_t getSerializedSize(const Document &doc) { - return doc.serialize()->getLength(); -} - -size_t getSerializedSizeHeader(const Document &doc) { - nbostream stream; - doc.serializeHeader(stream); - return stream.size(); -} - -size_t getSerializedSizeBody(const Document &doc) { - nbostream stream; - doc.serializeBody(stream); - return stream.size(); + return doc.serialize().size(); } TEST(DocumentTest, testGenerateSerializedFile) @@ -864,30 +860,21 @@ TEST(DocumentTest, testGenerateSerializedFile) map.put(StringFieldValue("foo2"), StringFieldValue("bar2")); doc.setValue("mapfield", map); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); + nbostream buf = doc.serialize(); const std::string serializedDir = TEST_PATH("../test/document/"); int fd = open((serializedDir + "/serializecpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.peek(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); - ByteBuffer hBuf(getSerializedSizeHeader(doc)); + vespalib::nbostream hBuf; doc.serializeHeader(hBuf); fd = open((serializedDir + "/serializecppsplit_header.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, hBuf.getBuffer(), hBuf.getPos()) != (ssize_t)hBuf.getPos()) { - throw vespalib::Exception("write failed"); - } - close(fd); - - ByteBuffer bBuf(getSerializedSizeBody(doc)); - doc.serializeBody(bBuf); - fd = open((serializedDir+ "/serializecppsplit_body.dat").c_str(), - O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, bBuf.getBuffer(), bBuf.getPos()) != (ssize_t)bBuf.getPos()) { + if (write(fd, hBuf.peek(), hBuf.size()) != (ssize_t)hBuf.size()) { throw vespalib::Exception("write failed"); } close(fd); @@ -895,62 +882,30 @@ TEST(DocumentTest, testGenerateSerializedFile) CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - ByteBuffer lz4buf(getSerializedSize(doc)); - - doc.serialize(lz4buf); - lz4buf.flip(); + nbostream lz4buf = doc.serialize(); fd = open((serializedDir + "/serializecpp-lz4-level9.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, lz4buf.getBufferAtPos(), lz4buf.getRemaining()) != (ssize_t)lz4buf.getRemaining()) { + if (write(fd, lz4buf.data(), lz4buf.size()) != (ssize_t)lz4buf.size()) { throw vespalib::Exception("write failed"); } close(fd); } -TEST(DocumentTest, testGetURIFromSerialized) -{ - TestDocRepo test_repo; - Document doc(*test_repo.getDocumentType("testdoctype1"), DocumentId("id:ns:testdoctype1::1")); - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - EXPECT_EQ( - vespalib::string("id:ns:testdoctype1::1"), - Document::getIdFromSerialized(*serialized).toString()); - - EXPECT_EQ(vespalib::string("testdoctype1"), - Document::getDocTypeFromSerialized( - test_repo.getTypeRepo(), - *serialized)->getName()); - } - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - Document doc2(test_repo.getTypeRepo(), *serialized, false, NULL); - EXPECT_EQ(vespalib::string("id:ns:testdoctype1::1"), doc2.getId().toString()); - EXPECT_EQ(vespalib::string("testdoctype1"), doc2.getType().getName()); - } -} - TEST(DocumentTest, testBogusserialize) { TestDocRepo test_repo; try { - auto buf = std::make_unique<ByteBuffer>("aoifjweprjwoejr203r+2+4r823++!",100); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("aoifjweprjwoejr203r+2+4r823++!",100); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing bogus data"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Unrecognized serialization version")); } try { - auto buf = std::make_unique<ByteBuffer>("",0); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("",0); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing empty buffer"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Buffer out of bounds")); @@ -967,24 +922,23 @@ TEST(DocumentTest, testCRC32) uint32_t crc = doc.calculateChecksum(); EXPECT_EQ(3987392271u, crc); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); + nbostream buf = doc.serialize(); int pos = 30; // Corrupt serialization. - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; // Create document. Byte corrupted above is in data area and // shouldn't fail deserialization. try { - Document doc2(test_repo.getTypeRepo(), *buf); - buf->setPos(0); + Document doc2(test_repo.getTypeRepo(), buf); + buf.rp(0); EXPECT_TRUE(crc != doc2.calculateChecksum()); } catch (document::DeserializeException& e) { EXPECT_TRUE(false); } // Return original value and retry - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; /// \todo TODO (was warning): Cannot test for in memory representation altered, as there is no syntax for getting internal refs to data from document. Add test when this is added. } @@ -1001,13 +955,12 @@ TEST(DocumentTest, testHasChanged) // Still changed after setting a value of course. EXPECT_TRUE(doc.hasChanged()); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); - + nbostream buf; + doc.serialize(buf); // Setting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); EXPECT_TRUE(!doc2.hasChanged()); doc2.set("headerval", 13); @@ -1015,16 +968,16 @@ TEST(DocumentTest, testHasChanged) } // Overwriting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.set("hstringval", "bla bla bla bla bla"); EXPECT_TRUE(doc2.hasChanged()); } // Clearing value tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.clear(); EXPECT_TRUE(doc2.hasChanged()); @@ -1032,35 +985,6 @@ TEST(DocumentTest, testHasChanged) // Add more tests here when we allow non-const refs to internals } -TEST(DocumentTest, testSplitSerialization) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - doc->set("headerval", 50); - - ByteBuffer buf(getSerializedSizeHeader(*doc)); - doc->serializeHeader(buf); - buf.flip(); - - ByteBuffer buf2(getSerializedSizeBody(*doc)); - doc->serializeBody(buf2); - buf2.flip(); - - EXPECT_EQ(size_t(65), buf.getLength()); - EXPECT_EQ(size_t(73), buf2.getLength()); - - Document headerDoc(testDocMan.getTypeRepo(), buf); - EXPECT_TRUE(headerDoc.hasValue("headerval")); - EXPECT_TRUE(!headerDoc.hasValue("content")); - - buf.setPos(0); - Document fullDoc(testDocMan.getTypeRepo(), buf, buf2); - EXPECT_TRUE(fullDoc.hasValue("headerval")); - EXPECT_TRUE(fullDoc.hasValue("content")); - - EXPECT_EQ(*doc, fullDoc); -} - TEST(DocumentTest, testSliceSerialize) { // Test that document doesn't need its own bytebuffer, such that we @@ -1076,17 +1000,15 @@ TEST(DocumentTest, testSliceSerialize) val.add(RawFieldValue("hei der", 7)); doc2->setValue(doc2->getField("rawarray"), val); - ByteBuffer buf(getSerializedSize(*doc) + getSerializedSize(*doc2)); - doc->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + nbostream buf = doc->serialize(); + EXPECT_EQ(getSerializedSize(*doc), buf.size()); doc2->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); - buf.flip(); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.size()); Document doc3(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc), buf.rp()); Document doc4(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.rp()); EXPECT_EQ(*doc, doc3); EXPECT_EQ(*doc2, doc4); @@ -1102,21 +1024,19 @@ TEST(DocumentTest, testCompression) doc->setValue("hstringval", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc->serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed = doc->serialize(); CompressionConfig oldCfg(doc->getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf_lz4 = doc->serialize(); - buf_lz4->flip(); + nbostream buf_lz4 = doc->serialize(); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(oldCfg); - EXPECT_TRUE(buf_lz4->getRemaining() < buf_uncompressed->getRemaining()); + EXPECT_TRUE(buf_lz4.size() < buf_uncompressed.size()); - Document doc_lz4(testDocMan.getTypeRepo(), *buf_lz4); + Document doc_lz4(testDocMan.getTypeRepo(), buf_lz4); EXPECT_EQ(*doc, doc_lz4); } @@ -1136,10 +1056,10 @@ TEST(DocumentTest, testCompressionConfigured) for (int i = 0; i < 8; ++i) { bigString += bigString; } doc_uncompressed.setValue("stringfield", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc_uncompressed.serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed; + doc_uncompressed.serialize(buf_uncompressed); - size_t uncompressedSize = buf_uncompressed->getRemaining(); + size_t uncompressedSize = buf_uncompressed.size(); DocumenttypesConfigBuilderHelper builder2; builder2.document(43, "serializetest", @@ -1151,22 +1071,22 @@ TEST(DocumentTest, testCompressionConfigured) 9, 99, 0)); DocumentTypeRepo repo2(builder2.config()); - Document doc(repo2, *buf_uncompressed); + Document doc(repo2, buf_uncompressed); - std::unique_ptr<ByteBuffer> buf_compressed = doc.serialize(); - buf_compressed->flip(); - size_t compressedSize = buf_compressed->getRemaining(); + nbostream buf_compressed; + doc.serialize(buf_compressed); + size_t compressedSize = buf_compressed.size(); EXPECT_TRUE(compressedSize < uncompressedSize); - Document doc2(repo2, *buf_compressed); + Document doc2(repo2, buf_compressed); - std::unique_ptr<ByteBuffer> buf_compressed2 = doc2.serialize(); - buf_compressed2->flip(); + nbostream buf_compressed2; + doc2.serialize(buf_compressed2); - EXPECT_EQ(compressedSize, buf_compressed2->getRemaining()); + EXPECT_EQ(compressedSize, buf_compressed2.size()); - Document doc3(repo2, *buf_compressed2); + Document doc3(repo2, buf_compressed2); EXPECT_EQ(doc2, doc_uncompressed); EXPECT_EQ(doc2, doc3); @@ -1198,59 +1118,31 @@ TEST(DocumentTest, testUnknownEntries) doc1.setValue(field3, IntFieldValue(3)); doc1.setValue(field4, IntFieldValue(4)); - uint32_t headerLen = getSerializedSizeHeader(doc1); - document::ByteBuffer header(headerLen); - doc1.serializeHeader(header); - header.flip(); - - uint32_t bodyLen = getSerializedSizeBody(doc1); - document::ByteBuffer body(bodyLen); - doc1.serializeBody(body); - body.flip(); - - uint32_t totalLen = getSerializedSize(doc1); - document::ByteBuffer total(totalLen); - doc1.serialize(total); - total.flip(); + vespalib::nbostream os; + doc1.serialize(os); Document doc2; - doc2.deserialize(repo, total); - - Document doc3; - doc3.deserializeHeader(repo, header); - doc3.deserializeBody(repo, body); + doc2.deserialize(repo, os); EXPECT_EQ(std::string( "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" "<int3>3</int3>\n" "<int4>4</int4>\n" "</document>"), doc2.toXml()); - EXPECT_EQ(std::string( - "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" - "<int3>3</int3>\n" - "<int4>4</int4>\n" - "</document>"), doc3.toXml()); EXPECT_EQ(3, doc2.getValue(field3)->getAsInt()); EXPECT_EQ(4, doc2.getValue(field4)->getAsInt()); - EXPECT_EQ(3, doc3.getValue(field3)->getAsInt()); - EXPECT_EQ(4, doc3.getValue(field4)->getAsInt()); // The fields are actually accessible as long as you ask with field of // correct type. EXPECT_TRUE(doc2.hasValue(field1)); EXPECT_TRUE(doc2.hasValue(field2)); - EXPECT_TRUE(doc3.hasValue(field1)); - EXPECT_TRUE(doc3.hasValue(field2)); EXPECT_EQ(1, doc2.getValue(field1)->getAsInt()); EXPECT_EQ(2, doc2.getValue(field2)->getAsInt()); - EXPECT_EQ(1, doc3.getValue(field1)->getAsInt()); - EXPECT_EQ(2, doc3.getValue(field2)->getAsInt()); EXPECT_EQ(size_t(2), doc2.getSetFieldCount()); - EXPECT_EQ(size_t(2), doc3.getSetFieldCount()); } TEST(DocumentTest, testAnnotationDeserialization) @@ -1280,14 +1172,15 @@ TEST(DocumentTest, testAnnotationDeserialization) int fd = open(TEST_PATH("data/serializejavawithannotations.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream1(buf.get(), len); + Document doc(repo, stream1); StringFieldValue strVal; EXPECT_TRUE(doc.getValue(doc.getField("story"), strVal)); @@ -1326,14 +1219,6 @@ TEST(DocumentTest, testAnnotationDeserialization) EXPECT_EQ((int64_t)2384LL, longVal.getAsLong()); } -TEST(DocumentTest, testGetSerializedSize) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - - EXPECT_EQ(getSerializedSize(*doc), doc->getSerializedSize()); -} - TEST(DocumentTest, testDeserializeMultiple) { TestDocRepo testDocRepo; diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 47a529adfc8..5543cb48ba4 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -43,14 +43,13 @@ namespace document { namespace { -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } nbostream serialize(const ValueUpdate & update) @@ -83,25 +82,25 @@ void testRoundtripSerialize(const UpdateType& update, const DataType &type) { } void -writeBufferToFile(const ByteBuffer &buf, const vespalib::string &fileName) +writeBufferToFile(const nbostream &buf, const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::out | std::ios::binary); - file.write(buf.getBuffer(), buf.getPos()); + file.write(buf.data(), buf.size()); assert(file.good()); file.close(); } -ByteBuffer::UP +nbostream readBufferFromFile(const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); auto size = file.tellg(); - auto result = std::make_unique<ByteBuffer>(size); file.seekg(0); - file.read(result->getBuffer(), size); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); + file.read(static_cast<char *>(buf.get()), size); assert(file.good()); file.close(); - return result; + return nbostream(std::move(buf), size); } } @@ -132,9 +131,8 @@ TEST(DocumentUpdateTest, testSimpleUsage) // Test that a document update can be serialized DocumentUpdate docUpdate(repo, *docType, DocumentId("id:ns:test::1")); docUpdate.addUpdate(fieldUpdateCopy); - ByteBuffer::UP docBuf = serializeHEAD(docUpdate); - docBuf->flip(); - auto docUpdateCopy(DocumentUpdate::createHEAD(repo, nbostream(docBuf->getBufferAtPos(), docBuf->getRemaining()))); + nbostream docBuf = serializeHEAD(docUpdate); + auto docUpdateCopy(DocumentUpdate::createHEAD(repo, docBuf)); // Create a test document Document doc(*docType, DocumentId("id:ns:test::1")); @@ -236,7 +234,7 @@ TEST(DocumentUpdateTest, testUpdateArray) // Create a document. TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); // Assign array field. ArrayFieldValue myarray(doc->getType().getField("tags").getDataType()); @@ -459,8 +457,7 @@ TEST(DocumentUpdateTest, testReadSerializedFile) const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); - auto buf = readBufferFromFile("data/serializeupdatejava.dat"); - nbostream is(buf->getBufferAtPos(), buf->getRemaining()); + auto is = readBufferFromFile("data/serializeupdatejava.dat"); DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, is)); DocumentUpdate& upd(*updp); @@ -539,8 +536,8 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 2))) .addUpdate(MapValueUpdate(StringFieldValue("foo"), ArithmeticValueUpdate(ArithmeticValueUpdate::Mul, 2)))); - ByteBuffer::UP buf(serializeHEAD(upd)); - writeBufferToFile(*buf, "data/serializeupdatecpp.dat"); + nbostream buf(serializeHEAD(upd)); + writeBufferToFile(buf, "data/serializeupdatecpp.dat"); } @@ -549,7 +546,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("headerval")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); // Assign a float value to an int field. DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); @@ -561,7 +558,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) update.applyTo(*doc); // Verify that the field is NOT set in the document. - EXPECT_EQ((document::FieldValue*)NULL, + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); } @@ -569,7 +566,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoParams) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(doc->getField("tags")).addUpdate(AssignValueUpdate())); @@ -1110,13 +1107,12 @@ struct TensorUpdateSerializeFixture { } void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { - ByteBuffer::UP buf = serializeHEAD(update); - writeBufferToFile(*buf, fileName); + nbostream buf = serializeHEAD(update); + writeBufferToFile(buf, fileName); } DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { - auto buf = readBufferFromFile(fileName); - nbostream stream(buf->getBufferAtPos(), buf->getRemaining()); + auto stream = readBufferFromFile(fileName); return DocumentUpdate::createHEAD(*repo, stream); } @@ -1196,10 +1192,9 @@ TEST(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeseriali { CreateIfNonExistentFixture f; - ByteBuffer::UP buf(serializeHEAD(*f.update)); - buf->flip(); + nbostream buf(serializeHEAD(*f.update)); - DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), *buf); + DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), buf); EXPECT_EQ(*f.update, *deserialized); EXPECT_TRUE(deserialized->getCreateIfNonExistent()); } @@ -1231,9 +1226,8 @@ TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) ArrayUpdateFixture f; auto buffer = serializeHEAD(*f.update); - buffer->flip(); - auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), *buffer); + auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), buffer); EXPECT_EQ(*f.update, *deserialized); } diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 82443f13716..213ac2e5432 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -19,6 +19,7 @@ #include <gtest/gtest.h> using vespalib::Identifiable; +using vespalib::nbostream; using namespace document::config_builder; namespace document { @@ -133,23 +134,21 @@ createTestDocument(const DocumentTypeRepo &repo) return doc; } -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { vespalib::nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { try{ - ByteBuffer::UP bb(serializeHEAD(a)); - bb->flip(); - DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb)); + auto bb(serializeHEAD(a)); + DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, bb)); - EXPECT_EQ(size_t(0), bb->getRemaining()); + EXPECT_EQ(size_t(0), bb.size()); EXPECT_EQ(a.getId().toString(), b->getId().toString()); EXPECT_EQ(a.getUpdates().size(), b->getUpdates().size()); for (size_t i(0); i < a.getUpdates().size(); i++) { @@ -157,8 +156,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { const FieldUpdate & ub = b->getUpdates()[i]; EXPECT_EQ(&ua.getField(), &ub.getField()); - EXPECT_EQ(ua.getUpdates().size(), - ub.getUpdates().size()); + EXPECT_EQ(ua.getUpdates().size(), ub.getUpdates().size()); for (size_t j(0); j < ua.getUpdates().size(); j++) { EXPECT_EQ(ua.getUpdates()[j]->getType(), ub.getUpdates()[j]->getType()); } @@ -1073,14 +1071,14 @@ TEST_F(FieldPathUpdateTestCase, testReadSerializedFile) int fd = open(TEST_PATH("data/serialize-fieldpathupdate-java.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, buf)); + DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, nbostream(std::move(buf), len))); DocumentUpdate& upd(*updp); DocumentUpdate::UP compare(createDocumentUpdateForSerialization(repo)); @@ -1094,11 +1092,11 @@ TEST_F(FieldPathUpdateTestCase, testGenerateSerializedFile) // Tests nothing, only generates a file for java test DocumentUpdate::UP upd(createDocumentUpdateForSerialization(repo)); - ByteBuffer::UP buf(serializeHEAD(*upd)); + nbostream buf(serializeHEAD(*upd)); int fd = open(TEST_PATH("data/serialize-fieldpathupdate-cpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.data(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index 58592e50cfe..8a5daf05f05 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -14,9 +14,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -89,20 +88,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Serialization Type t; - std::unique_ptr<ByteBuffer> buf(smallest.serialize()); - buf->flip(); - deserialize(*buf, t); + nbostream buf(smallest.serialize()); + deserialize(buf, t); EXPECT_EQ(smallest, t); buf = medium1.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(medium1, t); EXPECT_EQ(medium2, t); buf = largest.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(largest, t); // Assignment @@ -160,20 +156,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Test that a just deserialized value can be serialized again // (literals have lazy deserialization so behaves diff then value = "foo"; - std::unique_ptr<ByteBuffer> buf(value.serialize()); - buf->flip(); + nbostream buf(value.serialize()); Literal value2("Other"); - deserialize(*buf, value2); + deserialize(buf, value2); buf = value2.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(value, value2); // Verify that get value ref gives us ref within original bytebuffer // (operator== use above should not modify this) buf = value.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(size_t(3), value2.getValueRef().size()); // Zero termination diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 7feea4e51d2..f680b3e6cff 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -912,7 +912,7 @@ void DeserializedTensorDoc::setup(const DocumentTypeRepo &docTypeRepo, const vespalib::nbostream &blob) { vespalib::nbostream wrapStream(blob.peek(), blob.size()); - _doc = std::make_unique<Document>(docTypeRepo, wrapStream, nullptr); + _doc = std::make_unique<Document>(docTypeRepo, wrapStream); _fieldValue = _doc->getValue(tensor_field_name); } diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp index 76cf065a36a..9cf4b38be91 100644 --- a/document/src/tests/structfieldvaluetest.cpp +++ b/document/src/tests/structfieldvaluetest.cpp @@ -28,10 +28,9 @@ protected: namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value, const FixedTypeRepo &repo) +void deserialize(nbostream & stream, T &value, const FixedTypeRepo &repo) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); } @@ -61,13 +60,11 @@ TEST_F(StructFieldValueTest, testEmptyStruct) StructFieldValue value(type); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value == value2); } @@ -101,14 +98,12 @@ TEST_F(StructFieldValueTest, testStruct) EXPECT_EQ(2, value.getValue(longF)->getAsInt()); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); @@ -116,15 +111,15 @@ TEST_F(StructFieldValueTest, testStruct) // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); diff --git a/document/src/tests/testbytebuffer.cpp b/document/src/tests/testbytebuffer.cpp index 17807fb4ff5..7d412be5ba3 100644 --- a/document/src/tests/testbytebuffer.cpp +++ b/document/src/tests/testbytebuffer.cpp @@ -3,12 +3,14 @@ #include <vespa/document/util/stringutil.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/document/fieldvalue/serializablearray.h> -#include <iostream> -#include <vespa/vespalib/util/macro.h> #include <vespa/document/util/bufferexceptions.h> +#include <vespa/vespalib/util/macro.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <gtest/gtest.h> + using namespace document; +using vespalib::GrowableByteBuffer; namespace { @@ -22,111 +24,28 @@ void assign(S &lhs, const S &rhs) TEST(ByteBuffer_Test, test_constructors) { - ByteBuffer* simple=new ByteBuffer(); - delete simple; - - ByteBuffer* less_simple=new ByteBuffer("hei",3); - EXPECT_TRUE(strcmp(less_simple->getBufferAtPos(),"hei")==0); - delete less_simple; -} - -TEST(ByteBuffer_Test, test_assignment_operator) -{ - try { - ByteBuffer b1; - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); - - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - int test = 0; - b2.flip(); - b2.getInt(test); - EXPECT_EQ(1,test); - b2.getInt(test); - EXPECT_EQ(2,test); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - // Test Selfassignment == no change - // - assign(b2, b2); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - ByteBuffer b3; - // Empty - b2 = b3; - - EXPECT_EQ((size_t) 0,b2.getPos()); - EXPECT_EQ((size_t) 0,b2.getLength()); - EXPECT_EQ((size_t) 0,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } + ByteBuffer less_simple("hei",3); + EXPECT_TRUE(strcmp(less_simple.getBufferAtPos(),"hei")==0); } TEST(ByteBuffer_Test, test_copy_constructor) { try { - // Empty buffer first - ByteBuffer b1; - ByteBuffer b2(b1); - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); + GrowableByteBuffer gb(100); + gb.putInt(1); + gb.putInt(2); + ByteBuffer b1(gb.getBuffer(), gb.position()); ByteBuffer b2(b1); EXPECT_EQ(b1.getPos(),b2.getPos()); EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); int test = 0; - b2.flip(); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(1,test); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(2,test); } catch (std::exception &e) { @@ -134,327 +53,6 @@ TEST(ByteBuffer_Test, test_copy_constructor) } } -TEST(ByteBuffer_Test, test_slice) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer* slice = new ByteBuffer; - slice->sliceFrom(*newBuf, 0,newBuf->getLength()); - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice->getBufferAtPos(),"hei der")==0); - - ByteBuffer* slice2 = new ByteBuffer; - slice2->sliceFrom(*slice, 4, slice->getLength()); - delete slice; - slice = NULL; - - EXPECT_TRUE(strcmp(slice2->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2->getBuffer(),"hei der")==0); - delete slice2; - slice2 = NULL; - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - ByteBuffer* slice3=new ByteBuffer; - ByteBuffer* slice4=new ByteBuffer; - - slice3->sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice4->sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice3->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice3; - slice3 = NULL; - - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice4; - slice4 = NULL; -} - -TEST(ByteBuffer_Test, test_slice2) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer slice; - slice.sliceFrom(*newBuf, 0, newBuf->getLength()); - - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"hei der")==0); - - ByteBuffer slice2; - slice2.sliceFrom(slice, 4, slice.getLength()); - - EXPECT_TRUE(strcmp(slice2.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - - slice.sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice2.sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); -} - - -TEST(ByteBuffer_Test, test_putGetFlip) -{ - ByteBuffer* newBuf=new ByteBuffer(100); - - try { - newBuf->putInt(10); - int test; - newBuf->flip(); - - newBuf->getInt(test); - EXPECT_TRUE(test==10); - - newBuf->clear(); - newBuf->putDouble(3.35); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==sizeof(double)); - double test2; - newBuf->getDouble(test2); - EXPECT_TRUE(test2==3.35); - - newBuf->clear(); - newBuf->putBytes("heisann",8); - newBuf->putInt(4); - EXPECT_TRUE(newBuf->getPos()==12); - EXPECT_TRUE(newBuf->getLength()==100); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==12); - - char testStr[12]; - newBuf->getBytes(testStr, 8); - EXPECT_TRUE(strcmp(testStr,"heisann")==0); - newBuf->getInt(test); - EXPECT_TRUE(test==4); - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - delete newBuf; -} - - -TEST(ByteBuffer_Test, test_NumberEncodings) -{ - ByteBuffer* buf=new ByteBuffer(1024); - - // Check 0 - buf->putInt1_2_4Bytes(124); - buf->putInt2_4_8Bytes(124); - buf->putInt1_4Bytes(124); - // Check 1 - buf->putInt1_2_4Bytes(127); - buf->putInt2_4_8Bytes(127); - buf->putInt1_4Bytes(127); - // Check 2 - buf->putInt1_2_4Bytes(128); - buf->putInt2_4_8Bytes(128); - buf->putInt1_4Bytes(128); - // Check 3 - buf->putInt1_2_4Bytes(255); - buf->putInt2_4_8Bytes(255); - buf->putInt1_4Bytes(255); - // Check 4 - buf->putInt1_2_4Bytes(256); - buf->putInt2_4_8Bytes(256); - buf->putInt1_4Bytes(256); - // Check 5 - buf->putInt1_2_4Bytes(0); - buf->putInt2_4_8Bytes(0); - buf->putInt1_4Bytes(0); - // Check 6 - buf->putInt1_2_4Bytes(1); - buf->putInt2_4_8Bytes(1); - buf->putInt1_4Bytes(1); - - // Check 7 - try { - buf->putInt1_2_4Bytes(0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - buf->putInt2_4_8Bytes(0x7FFFFFFFll); - buf->putInt1_4Bytes(0x7FFFFFFF); - - try { - buf->putInt2_4_8Bytes(0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - buf->putInt1_2_4Bytes(0x7FFF); - // Check 8 - buf->putInt2_4_8Bytes(0x7FFFll); - buf->putInt1_4Bytes(0x7FFF); - buf->putInt1_2_4Bytes(0x7F); - // Check 9 - buf->putInt2_4_8Bytes(0x7Fll); - buf->putInt1_4Bytes(0x7F); - - try { - buf->putInt1_2_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt1_2_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - uint32_t endWritePos = buf->getPos(); - buf->setPos(0); - - int32_t tmp32; - int64_t tmp64; - - // Check 0 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)124, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - // Check 1 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)127, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - // Check 2 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)128, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - // Check 3 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)255, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - // Check 4 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)256, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - // Check 5 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - // Check 6 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)1, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - // Check 7 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFFFFFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFFFFFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - // Check 8 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - // Check 9 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7F, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - - uint32_t endReadPos = buf->getPos(); - EXPECT_EQ(endWritePos, endReadPos); - - delete buf; -} - -TEST(ByteBuffer_Test, test_NumberLengths) -{ - ByteBuffer b; - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(127)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(128)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(129)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(255)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(256)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(0x7FFFFFFF)); - - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(0)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(1)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(4)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(31)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(126)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(32767)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32768)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32769)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(1030493)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(0x3FFFFFFF)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000000)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000001)); - - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(16383)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16384)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16385)); -} - TEST(ByteBuffer_Test, test_SerializableArray) { SerializableArray array; @@ -464,6 +62,6 @@ TEST(ByteBuffer_Test, test_SerializableArray) EXPECT_EQ(4ul, array.get(0).size()); EXPECT_EQ(copy.get(0).size(), array.get(0).size()); EXPECT_TRUE(copy.get(0).c_str() != array.get(0).c_str()); - EXPECT_EQ(0, strcmp(copy.get(0).c_str(), array.get(0).c_str())); + EXPECT_EQ(0, strncmp(copy.get(0).c_str(), array.get(0).c_str(), 4)); EXPECT_EQ(16ul, sizeof(SerializableArray::Entry)); } diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp index 1ec52791a4a..71dc9f1d1a9 100644 --- a/document/src/tests/weightedsetfieldvaluetest.cpp +++ b/document/src/tests/weightedsetfieldvaluetest.cpp @@ -16,9 +16,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -94,26 +93,25 @@ TEST(WeightedSetFieldValueTest, testWeightedSet) EXPECT_EQ(6, value.get(IntFieldValue(3))); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); WeightedSetFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(buffer, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); EXPECT_EQ(size_t(3), value2.size()); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 29414c901f8..969684f104b 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -8,12 +8,10 @@ #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <sstream> -#include <limits> using vespalib::nbostream; using vespalib::make_string; @@ -24,10 +22,6 @@ using namespace vespalib::xml; namespace document { namespace { -bool isLegalVersion(uint16_t version) { - return (6 <= version) && (version <= 8); -} - void documentTypeError(vespalib::stringref name) __attribute__((noinline)); void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) __attribute__((noinline)); @@ -93,22 +87,13 @@ Document::Document(const DataType &type, DocumentId documentId) } } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserialize(repo, buffer); -} - void Document::setRepo(const DocumentTypeRepo& repo) { _fields.setRepo(repo); } -Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), +Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is) + : StructuredFieldValue(*DataType::DOCUMENT), _id(), _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), _lastModified(0) @@ -116,33 +101,6 @@ Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const deserialize(repo, is); } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - if (!includeContent) { - const DocumentType *newDocType = deserializeDocHeaderAndType(repo, buffer, _id, static_cast<const DocumentType*>(anticipatedType)); - if (newDocType) { - setType(*newDocType); - } - } else { - deserialize(repo, buffer); - } -} - - -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserializeHeader(repo, header); - deserializeBody(repo, body); -} - Document::~Document() = default; const DocumentType& @@ -170,30 +128,6 @@ Document::hasChanged() const return _fields.hasChanged(); } -DocumentId -Document::getIdFromSerialized(ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - deserializeDocHeader(buf, retVal); - buf.setPos(position); - - return retVal; -} - -const DocumentType * -Document::getDocTypeFromSerialized(const DocumentTypeRepo& repo, ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - const DocumentType *docType(deserializeDocHeaderAndType(repo, buf, retVal, nullptr)); - buf.setPos(position); - - return docType; -} - FieldValue& Document::assign(const FieldValue& value) { @@ -274,120 +208,11 @@ Document::calculateChecksum() const return calculator.checksum() ^ _fields.calculateChecksum(); } -const DocumentType * -Document::deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, DocumentId& id, - const DocumentType * docType) -{ - deserializeDocHeader(buffer, id); - - vespalib::stringref docTypeName(buffer.getBufferAtPos()); - buffer.incPos(docTypeName.size() + 1); // Skip 0-byte too - { - int16_t docTypeVersion; // version not supported anymore - buffer.getShortNetwork(docTypeVersion); - } - const DocumentType *docTypeNew = nullptr; - - if (! ((docType != nullptr) && (docType->getName() == docTypeName))) { - docTypeNew = repo.getDocumentType(docTypeName); - if (!docTypeNew) { - throw DocumentTypeNotFoundException(docTypeName, VESPA_STRLOC); - } - } - return docTypeNew; -} - -namespace { -[[noreturn]] void versionError(uint16_t version) __attribute__((noinline)); -[[noreturn]] void mainDocumentError(int64_t len) __attribute__((noinline)); -[[noreturn]] void notEnoughDocumentError(int32_t len, int64_t remaining) __attribute__((noinline)); - -void versionError(uint16_t version) { - throw DeserializeException(make_string( "Unrecognized serialization version %d", version), VESPA_STRLOC); -} - -void mainDocumentError(int64_t len) { - throw DeserializeException(make_string( - "Document lengths past %i is not supported. Corrupt data said length is %" PRId64 " bytes", - std::numeric_limits<int>::max(), len), VESPA_STRLOC); -} - -void notEnoughDocumentError(int32_t len, int64_t remaining) { - throw DeserializeException(make_string( "Buffer said document length is %d bytes, but only %" PRId64 " bytes remain in buffer", len, remaining)); -} - -} - -void -Document::deserializeDocHeader(ByteBuffer& buffer, DocumentId& id) { - int16_t version; - int32_t len; - buffer.getShortNetwork(version); - - if ( ! isLegalVersion(version) ) { - versionError(version); - } else if (version < 7) { - int64_t tmpLen = 0; - buffer.getInt2_4_8Bytes(tmpLen); - if (tmpLen > std::numeric_limits<int>::max()) { - mainDocumentError(tmpLen); - } else { - len = static_cast<int32_t>(tmpLen) - - ByteBuffer::getSerializedSize2_4_8Bytes(tmpLen) - - sizeof(uint16_t); - } - } else { - buffer.getIntNetwork(len); - } - - if (len > (long)buffer.getRemaining()) { - notEnoughDocumentError(len, buffer.getRemaining()); - } else { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - id = DocumentId(stream); - buffer.incPos(stream.rp()); - unsigned char contentByte; - buffer.getByte(contentByte); - } -} - -void Document::serializeHeader(ByteBuffer& buffer) const { - nbostream stream; - serializeHeader(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - void Document::serializeHeader(nbostream& stream) const { VespaDocumentSerializer serializer(stream); serializer.write(*this, WITHOUT_BODY); } -void Document::serializeBody(ByteBuffer& buffer) const { - nbostream stream; - serializeBody(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -bool Document::hasBodyField() const { - for (document::StructuredFieldValue::const_iterator it(getFields().begin()), mt(getFields().end()); - it != mt; - ++it) - { - if ( ! it.field().isHeaderField() ) { - return true; - } - } - return false; -} - -void Document::serializeBody(nbostream& stream) const { - if (hasBodyField()) { - VespaDocumentSerializer serializer(stream); - serializer.write(_fields, BodyFields()); - } -} - void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os) { VespaDocumentDeserializer deserializer(repo, os, 0); try { @@ -397,38 +222,19 @@ void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & o } } -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& data) { - nbostream stream(data.getBufferAtPos(), data.getRemaining()); - deserialize(repo, stream); - data.incPos(data.getRemaining() - stream.size()); -} - -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body) { +void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & header, vespalib::nbostream & body) { deserializeHeader(repo, header); deserializeBody(repo, body); } -void Document::deserializeHeader(const DocumentTypeRepo& repo, - ByteBuffer& header) { - nbostream stream(header.getBufferAtPos(), header.getRemaining()); +void Document::deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { VespaDocumentDeserializer deserializer(repo, stream, 0); deserializer.read(*this); - header.incPos(header.getRemaining() - stream.size()); } -void Document::deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body) { - nbostream body_stream(body.getBufferAtPos(), body.getRemaining()); - VespaDocumentDeserializer - body_deserializer(repo, body_stream, getFields().getVersion()); - body_deserializer.readStructNoReset(getFields()); - body.incPos(body.getRemaining() - body_stream.size()); -} - -size_t -Document::getSerializedSize() const -{ - // Temporary non-optimal (but guaranteed correct) implementation. - return serialize()->getLength(); +void Document::deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { + VespaDocumentDeserializer deserializer(repo, stream, getFields().getVersion()); + deserializer.readStructNoReset(getFields()); } StructuredFieldValue::StructuredIterator::UP diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h index 3e1ec0da3e6..f20ef969e0c 100644 --- a/document/src/vespa/document/fieldvalue/document.h +++ b/document/src/vespa/document/fieldvalue/document.h @@ -27,10 +27,11 @@ private: DocumentId _id; StructFieldValue _fields; - // To avoid having to return another container object out of docblocks - // the meta data has been added to document. This will not be serialized - // with the document and really doesn't belong here! + // To avoid having to return another container object out of docblocks + // the meta data has been added to document. This will not be serialized + // with the document and really doesn't belong here! int64_t _lastModified; + public: typedef std::unique_ptr<Document> UP; typedef std::shared_ptr<Document> SP; @@ -42,13 +43,7 @@ public: Document(); Document(const Document&); Document(const DataType &, DocumentId id); - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType = nullptr); - Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream, const DataType *anticipatedType = nullptr); - /** - Constructor to deserialize only document and type from a buffer. Only relevant if includeContent is false. - */ - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType); - Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType = nullptr); + Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream); ~Document() override; void setRepo(const DocumentTypeRepo & repo); @@ -86,21 +81,6 @@ public: bool hasChanged() const override; - /** - * Returns a pointer to the Id of a serialized document, without performing - * the deserialization. buffer must point to the start position of the - * serialization. If the buffer doesn't have enough data remaining to have - * a legal Id in it, method returns NULL. - */ - static DocumentId getIdFromSerialized(ByteBuffer&); - - /** - * Returns a pointer to the document type of a serialized header, without - * performing the deserialization. Buffer must point to the start position - * of the serialization. - */ - static const DocumentType *getDocTypeFromSerialized(const DocumentTypeRepo&, ByteBuffer&); - // FieldValue implementation. FieldValue& assign(const FieldValue&) override; int compare(const FieldValue& other) const override; @@ -108,22 +88,12 @@ public: void printXml(XmlOutputStream& out) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - // Specialized serialization functions - void serializeHeader(ByteBuffer& buffer) const; + // Specialized serialization functions, Only used for testing legacy stuff void serializeHeader(vespalib::nbostream& stream) const; - void serializeBody(ByteBuffer& buffer) const; - void serializeBody(vespalib::nbostream& stream) const; - - /** Deserialize document contained in given bytebuffer. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& data); void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os); /** Deserialize document contained in given bytebuffers. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& body, ByteBuffer& header); - void deserializeHeader(const DocumentTypeRepo& repo, ByteBuffer& header); - void deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body); - - size_t getSerializedSize() const; + void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & body, vespalib::nbostream & header); /** Undo fieldvalue's toXml override for document. */ std::string toXml() const { return toXml(""); } @@ -137,18 +107,14 @@ public: void setFieldValue(const Field& field, FieldValue::UP data) override; private: - bool hasBodyField() const; + void deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & header); + void deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & body); bool hasFieldValue(const Field& field) const override { return _fields.hasValue(field); } void removeFieldValue(const Field& field) override { _fields.remove(field); } FieldValue::UP getFieldValue(const Field& field) const override { return _fields.getValue(field); } bool getFieldValue(const Field& field, FieldValue& value) const override { return _fields.getValue(field, value); } StructuredIterator::UP getIterator(const Field* first) const override; - - static void deserializeDocHeader(ByteBuffer& buffer, DocumentId& id); - static const DocumentType *deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, - DocumentId& id, const DocumentType * docType); }; } // document diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 999747688eb..7737d03b45a 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -35,22 +35,11 @@ void FieldValue::serialize(nbostream &stream) const { serializer.write(*this); } -void FieldValue::serialize(ByteBuffer& buffer) const { +nbostream +FieldValue::serialize() const { nbostream stream; serialize(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -std::unique_ptr<ByteBuffer> FieldValue::serialize() const { - nbostream stream; - serialize(stream); - - nbostream::Buffer buf; - stream.swap(buf); - size_t sz = buf.size(); - auto bb = std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); - bb->setPos(sz); - return bb; + return stream; } size_t @@ -58,7 +47,7 @@ FieldValue::hash() const { vespalib::nbostream os; serialize(os); - return vespalib::hashValue(os.c_str(), os.size()) ; + return vespalib::hashValue(os.data(), os.size()) ; } bool diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index a152f74cc09..9ed8d522b17 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -19,15 +19,11 @@ #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/util/polymorphicarraybase.h> -namespace vespalib { - class nbostream; -} +namespace vespalib { class nbostream; } namespace document { -namespace fieldvalue { - class IteratorHandler; -} +namespace fieldvalue { class IteratorHandler; } class ByteBuffer; class DataType; @@ -73,8 +69,7 @@ public: virtual bool isA(const FieldValue& other) const; void serialize(vespalib::nbostream &stream) const; - void serialize(ByteBuffer& buffer) const; - std::unique_ptr<ByteBuffer> serialize() const; + vespalib::nbostream serialize() const; /** * Compares this fieldvalue with another fieldvalue. diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 5dfd8eff891..722904659ef 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -26,15 +26,30 @@ public: } -SerializableArray::Statistics SerializableArray::_stats; - SerializableArray::SerializableArray() : _serializedCompression(CompressionConfig::NONE), _uncompressedLength(0) { } -serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { +SerializableArray::SerializableArray(EntryMap entries, ByteBuffer::UP buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length) + : _entries(std::move(entries)), + _owned(), + _serializedCompression(comp_type) +{ + + if (CompressionConfig::isCompressed(_serializedCompression)) { + _compSerData = std::move(buffer); + _uncompressedLength = uncompressed_length; + } else { + _uncompressedLength = buffer->getRemaining(); + _uncompSerData = std::move(buffer); + } +} + +serializablearray::BufferMap & +ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { if (!owned) { owned = std::make_unique<serializablearray::BufferMap>(); } @@ -45,8 +60,8 @@ SerializableArray::SerializableArray(const SerializableArray& other) : Cloneable(), _entries(other._entries), _owned(), - _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : NULL), - _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : NULL), + _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : nullptr), + _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : nullptr), _serializedCompression(other._serializedCompression), _uncompressedLength(other._uncompressedLength) { @@ -66,17 +81,6 @@ SerializableArray::SerializableArray(const SerializableArray& other) } } -void -SerializableArray::swap(SerializableArray& other) -{ - _entries.swap(other._entries); - _owned.swap(other._owned); - std::swap(_uncompSerData, other._uncompSerData); - std::swap(_compSerData, other._compSerData); - std::swap(_serializedCompression, other._serializedCompression); - std::swap(_uncompressedLength, other._uncompressedLength); -} - void SerializableArray::clear() { _entries.clear(); @@ -86,9 +90,7 @@ void SerializableArray::clear() _uncompressedLength = 0; } -SerializableArray::~SerializableArray() -{ -} +SerializableArray::~SerializableArray() = default; void SerializableArray::invalidate() @@ -102,7 +104,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer) maybeDecompress(); Entry e(id, buffer->getRemaining(), buffer->getBuffer()); ensure(_owned)[id] = std::move(buffer); - EntryMap::iterator it = find(id); + auto it = find(id); if (it == _entries.end()) { _entries.push_back(e); } else { @@ -139,7 +141,7 @@ SerializableArray::get(int id) const { vespalib::ConstBufferRef buf; if ( !maybeDecompressAndCatch() ) { - EntryMap::const_iterator found = find(id); + auto found = find(id); if (found != _entries.end()) { const Entry& entry = *found; @@ -168,7 +170,7 @@ void SerializableArray::clear(int id) { maybeDecompress(); - EntryMap::iterator it = find(id); + auto it = find(id); if (it != _entries.end()) { _entries.erase(it); if (_owned) { @@ -193,7 +195,7 @@ SerializableArray::deCompress() // throw (DeserializeException) _uncompSerData = std::move(_compSerData); LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); } else { - ByteBuffer::UP newSerialization(new ByteBuffer(_uncompressedLength)); + auto newSerialization = std::make_unique<ByteBuffer>(vespalib::alloc::Alloc::alloc(_uncompressedLength), _uncompressedLength); vespalib::DataBuffer unCompressed(newSerialization->getBuffer(), newSerialization->getLength()); unCompressed.clear(); try { @@ -210,35 +212,16 @@ SerializableArray::deCompress() // throw (DeserializeException) if (unCompressed.getDataLen() != (size_t)_uncompressedLength) { throw DeserializeException( - make_string("Did not decompress to the expected length: had %zu, wanted %d, got %zu", + make_string("Did not decompress to the expected length: had %u, wanted %d, got %zu", _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()), VESPA_STRLOC); } assert(newSerialization->getBuffer() == unCompressed.getData()); - newSerialization->setLimit(_uncompressedLength); _uncompSerData = std::move(newSerialization); LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); } } -void SerializableArray::assign(EntryMap & entries, - ByteBuffer::UP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length) -{ - _serializedCompression = comp_type; - - _entries.clear(); - _entries.swap(entries); - if (CompressionConfig::isCompressed(_serializedCompression)) { - _compSerData.reset(buffer.release()); - _uncompressedLength = uncompressed_length; - } else { - _uncompressedLength = buffer->getRemaining(); - _uncompSerData.reset(buffer.release()); - } -} - vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2f7d65938aa..1e766599dff 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -26,7 +26,6 @@ namespace document { -class SerializableArrayIterator; class ByteBuffer; namespace serializablearray { @@ -36,25 +35,6 @@ namespace serializablearray { class SerializableArray : public vespalib::Cloneable { public: - // Counts set during serialization, in order to provide metrics for how - // often we use cached version, and how often we compress. - struct Statistics { - uint64_t _usedCachedSerializationCount; - uint64_t _compressedDocumentCount; - uint64_t _compressionDidntHelpCount; - uint64_t _uncompressableCount; - uint64_t _serializedUncompressed; - uint64_t _inputWronglySerialized; - - Statistics() - : _usedCachedSerializationCount(0), - _compressedDocumentCount(0), - _compressionDidntHelpCount(0), - _uncompressableCount(0), - _serializedUncompressed(0), - _inputWronglySerialized(0) {} - }; - /** * Contains the id of a field, the size and a buffer reference that is either * a relative offset to a common buffer, or the buffer itself it it is not. @@ -98,12 +78,6 @@ public: static const uint32_t ReservedId = 100; static const uint32_t ReservedIdUpper = 128; - -private: - static Statistics _stats; - -public: - static Statistics& getStatistics() { return _stats; } using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; @@ -111,9 +85,9 @@ public: using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); - virtual ~SerializableArray(); - - void swap(SerializableArray& other); + SerializableArray(EntryMap entries, ByteBufferUP buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length); + ~SerializableArray() override; /** * Stores a value in the array. @@ -141,9 +115,6 @@ public: /** @return Returns true if the given ID is Set in the array. */ bool has(int id) const; - /** @return Number of elements in array */ - bool hasAnyElems() const { return !_entries.empty(); } - /** * clears an attribute. * @@ -157,16 +128,6 @@ public: CompressionConfig::Type getCompression() const { return _serializedCompression; } CompressionInfo getCompressionInfo() const; - /** - * Sets the serialized data that is the basis for this object's - * content. This is used by deserialization. Any existing entries - * are cleared. - */ - void assign(EntryMap &entries, - ByteBufferUP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length); - bool empty() const { return _entries.empty(); } const ByteBuffer* getSerializedBuffer() const { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 87c8d09648c..b861537f6f7 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -48,15 +48,14 @@ StructFieldValue::Chunks::~Chunks() = default; void StructFieldValue::Chunks::push_back(SerializableArray::UP item) { - assert(_sz < 2); - _chunks[_sz++].reset(item.release()); + assert(size() < 2); + _chunks[size()] = std::move(item); } void StructFieldValue::Chunks::clear() { _chunks[0].reset(); _chunks[1].reset(); - _sz = 0; } const StructDataType & @@ -81,13 +80,11 @@ StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, _doc_type = &repo.getDocumentType(); _version = version; - _chunks.push_back(std::make_unique<SerializableArray>()); - _chunks.back().assign(fm, std::move(buffer), comp_type, uncompressed_length); + _chunks.push_back(std::make_unique<SerializableArray>(std::move(fm), std::move(buffer), comp_type, uncompressed_length)); _hasChanged = false; } -bool StructFieldValue::serializeField(int field_id, uint16_t version, - FieldValueWriter &writer) const { +bool StructFieldValue::serializeField(int field_id, uint16_t version, FieldValueWriter &writer) const { if (version == _version) { for (int i = _chunks.size() - 1; i >= 0; --i) { vespalib::ConstBufferRef buf = _chunks[i].get(field_id); @@ -183,9 +180,9 @@ StructFieldValue::getFieldValue(const Field& field) const if (buf.size() != 0) { nbostream stream(buf.c_str(), buf.size()); FieldValue::UP value(field.getDataType().createFieldValue()); - if ((_repo == NULL) && (_doc_type != NULL)) { - std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); - createFV(*value, *tmpRepo, stream, *_doc_type, _version); + if ((_repo == nullptr) && (_doc_type != nullptr)) { + DocumentTypeRepo tmpRepo(*_doc_type); + createFV(*value, tmpRepo, stream, *_doc_type, _version); } else { createFV(*value, *_repo, stream, *_doc_type, _version); } @@ -216,7 +213,7 @@ StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const vespalib::ConstBufferRef buf = getRawField(fieldId); if (buf.size() > 0) { nbostream_longlivedbuf stream(buf.c_str(), buf.size()); - if ((_repo == NULL) && (_doc_type != NULL)) { + if ((_repo == nullptr) && (_doc_type != nullptr)) { std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); createFV(value, *tmpRepo, stream, *_doc_type, _version); } else { @@ -239,17 +236,29 @@ StructFieldValue::hasFieldValue(const Field& field) const return false; } +namespace { + +std::unique_ptr<ByteBuffer> +serializeDoc(const FieldValue & fv) { + nbostream stream = fv.serialize(); + nbostream::Buffer buf; + stream.swap(buf); + size_t sz = buf.size(); + return std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); +} + +} void StructFieldValue::setFieldValue(const Field& field, FieldValue::UP value) { int fieldId = field.getId(); - std::unique_ptr<ByteBuffer> serialized(value->serialize()); - serialized->flip(); + + std::unique_ptr<ByteBuffer> serialized = serializeDoc(*value); if (_chunks.empty()) { - _chunks.push_back(SerializableArray::UP(new SerializableArray())); + _chunks.push_back(std::make_unique<SerializableArray>()); } - _chunks.back().set(fieldId, std::move(serialized)); + _chunks[0].set(fieldId, std::move(serialized)); _hasChanged = true; } @@ -274,7 +283,7 @@ StructFieldValue::clear() FieldValue& StructFieldValue::assign(const FieldValue& value) { - const StructFieldValue& other(dynamic_cast<const StructFieldValue&>(value)); + const auto & other(dynamic_cast<const StructFieldValue&>(value)); return operator=(other); } @@ -285,7 +294,7 @@ StructFieldValue::compare(const FieldValue& otherOrg) const if (comp != 0) { return comp; } - const StructFieldValue& other = static_cast<const StructFieldValue&>(otherOrg); + const auto & other = static_cast<const StructFieldValue&>(otherOrg); std::vector<int> a; getRawFieldIds(a); @@ -315,9 +324,9 @@ StructFieldValue::compare(const FieldValue& otherOrg) const uint32_t StructFieldValue::calculateChecksum() const { - ByteBuffer::UP buffer(serialize()); + nbostream buffer(serialize()); vespalib::crc_32_type calculator; - calculator.process_bytes(buffer->getBuffer(), buffer->getPos()); + calculator.process_bytes(buffer.peek(), buffer.size()); return calculator.checksum(); } @@ -386,7 +395,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { std::vector<int> _ids; std::vector<int>::iterator _cur; - FieldIterator(const StructFieldValue& s) + explicit FieldIterator(const StructFieldValue& s) : _struct(s), _ids(), _cur(_ids.begin()) @@ -412,7 +421,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { LOG(debug, "struct data type: %s", _struct.getType().toString(true).c_str()); } } - return 0; + return nullptr; } }; @@ -421,10 +430,10 @@ StructFieldValue::getIterator(const Field* toFind) const { StructuredIterator::UP ret; - FieldIterator *fi = new FieldIterator(*this); + auto *fi = new FieldIterator(*this); ret.reset(fi); - if (toFind != NULL) { + if (toFind != nullptr) { fi->skipTo(toFind->getId()); } return ret; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index b5b15e9dfce..30500229813 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -27,19 +27,16 @@ class StructFieldValue : public StructuredFieldValue public: class Chunks { public: - Chunks() : _sz(0) { } + Chunks() { } ~Chunks(); SerializableArray & operator [] (size_t i) { return *_chunks[i]; } const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; } VESPA_DLL_LOCAL void push_back(SerializableArray::UP item); - SerializableArray & back() { return *_chunks[_sz-1]; } - const SerializableArray & back() const { return *_chunks[_sz-1]; } - size_t size() const { return _sz; } - bool empty() const { return _sz == 0; } + size_t size() const { return _chunks[1] ? 2 : _chunks[0] ? 1 : 0; } + bool empty() const { return !_chunks[0]; } VESPA_DLL_LOCAL void clear(); private: SerializableArray::CP _chunks[2]; - size_t _sz; }; private: Chunks _chunks; diff --git a/document/src/vespa/document/serialization/util.h b/document/src/vespa/document/serialization/util.h index cbcc2c97017..05953f1de2b 100644 --- a/document/src/vespa/document/serialization/util.h +++ b/document/src/vespa/document/serialization/util.h @@ -97,32 +97,5 @@ void putInt2_4_8Bytes(Output &out, uint64_t val) { } } -inline uint32_t sizeOfInt1_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt1_2_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else if (val < 0x4000) { - return 2; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt2_4_8Bytes(uint64_t val) { - if (val < 0x8000) { - return 2; - } else if (val < 0x40000000) { - return 4; - } else { - return 8; - } -} } // namespace document diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 570a51907da..4d60befba52 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -293,10 +293,10 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { using vespalib::compression::compress; - vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); + vespalib::ConstBufferRef buf(stream.data(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, - vespalib::ConstBufferRef(stream.c_str(), stream.size()), + vespalib::ConstBufferRef(stream.data(), stream.size()), compressed_data, false); if (compressedType != config.type || ! compressionSufficient(config, stream.size(), compressed_data.getDataLen())) diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index b43b7a59c5b..f289e6a8f27 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -7,7 +7,6 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/bufferexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> @@ -231,20 +230,18 @@ DocumentUpdate::serializeFlags(int size_) const } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream && stream) { - vespalib::nbostream is(buffer.getBufferAtPos(), buffer.getRemaining()); auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, is); - buffer.setPos(buffer.getPos() + is.rp()); + update->initHEAD(repo, std::move(stream)); return update; } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream stream) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, std::move(stream)); + update->initHEAD(repo, stream); return update; } diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index a672d028f35..fb3d6c0f7a3 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -32,6 +32,7 @@ namespace document { +class ByteBuffer; class Document; class VespaDocumentSerializer; /** @@ -50,8 +51,8 @@ public: /** * Create new style document update, possibly with field path updates. */ - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream stream); - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, ByteBuffer & buffer); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); DocumentUpdate(); /** @@ -124,8 +125,8 @@ private: bool _needHardReserialize; int deserializeFlags(int sizeAndFlags); - void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void deserializeBody(const DocumentTypeRepo &repo, vespalib::nbostream &stream); void lazyDeserialize(const DocumentTypeRepo & repo, vespalib::nbostream & stream); void ensureDeserialized() const; diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 3e19420d0d0..0dbf8e5cc9e 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -7,7 +7,6 @@ namespace document { -class ByteBuffer; class DocumentTypeRepo; class Field; class FieldValue; @@ -35,7 +34,7 @@ public: using SP = std::shared_ptr<FieldPathUpdate>; using CP = vespalib::CloneablePtr<FieldPathUpdate>; - ~FieldPathUpdate(); + ~FieldPathUpdate() override; enum FieldPathUpdateType { Add = IDENTIFIABLE_CLASSID(AddFieldPathUpdate), diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 8cb148abe25..48ca7bd36d7 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp printable.cpp - serializable.cpp + serializableexceptions.cpp stringutil.cpp DEPENDS AFTER diff --git a/document/src/vespa/document/util/bufferexceptions.h b/document/src/vespa/document/util/bufferexceptions.h index 8a3215f6c79..aee7f3ae568 100644 --- a/document/src/vespa/document/util/bufferexceptions.h +++ b/document/src/vespa/document/util/bufferexceptions.h @@ -15,13 +15,5 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(BufferOutOfBoundsException) }; -class InputOutOfRangeException : public vespalib::IoException { -public: - InputOutOfRangeException(const vespalib::string& msg, - const vespalib::string& location = ""); - - VESPA_DEFINE_EXCEPTION_SPINE(InputOutOfRangeException) -}; - } diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp index ccbc2bc7790..0fbc6ca1c63 100644 --- a/document/src/vespa/document/util/bytebuffer.cpp +++ b/document/src/vespa/document/util/bytebuffer.cpp @@ -12,732 +12,160 @@ #include <sstream> #include <arpa/inet.h> -#define LOG_DEBUG1(a) -// Enable this macros instead to see what bytebuffer calls come -//#define LOG_DEBUG1(a) std::cerr << "ByteBuffer(" << ((void*) this) << " " << a << ")\n"; - -#define LOG_DEBUG2(a,b) LOG_DEBUG1(vespalib::make_string(a,b)); -#define LOG_DEBUG3(a,b,c) LOG_DEBUG1(vespalib::make_string(a,b,c)); -#define LOG_DEBUG4(a,b,c,d) LOG_DEBUG1(vespalib::make_string(a,b,c,d)); - using vespalib::alloc::Alloc; +using vespalib::make_string; namespace document { -VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(InputOutOfRangeException); +namespace { -vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { - vespalib::asciistream ost; - ost << pos << " > " << len; - return ost.str(); -} +static void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline, noreturn)); -BufferOutOfBoundsException::BufferOutOfBoundsException( - size_t pos, size_t len, const vespalib::string& location) - : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) +void throwOutOfBounds(size_t want, size_t has) { + throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); } -InputOutOfRangeException::InputOutOfRangeException( - const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::INTERNAL_FAILURE, location, 1) -{ } -ByteBuffer::ByteBuffer() : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(NULL, 0); - LOG_DEBUG1("Created empty bytebuffer"); +#if defined(__i386__) || defined(__x86_64__) + +template<typename T> +void +ByteBuffer::getDoubleLongNetwork(T &val) { + //TODO: Change this if we move to big-endian hardware + if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { + throwOutOfBounds(sizeof(T), getRemaining()); + } + + auto * data = reinterpret_cast<unsigned char*>(&val); + for (int i=sizeof(T)-1; i>=0; --i) { + getByte(data[i]); + } } -ByteBuffer::ByteBuffer(size_t len) : - ByteBuffer(Alloc::alloc(len), len) +#else +#error "getDoubleLongNetwork is undefined for this arcitecture" +#endif + +VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); + +vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { + vespalib::asciistream ost; + ost << pos << " > " << len; + return ost.str(); +} + +BufferOutOfBoundsException::BufferOutOfBoundsException(size_t pos, size_t len, const vespalib::string& location) + : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) { } -ByteBuffer::ByteBuffer(const char* buffer, size_t len) : - _buffer(NULL), - _len(0), +ByteBuffer::ByteBuffer(const char* buffer, uint32_t len) : + _buffer(const_cast<char *>(buffer)), + _len(len), _pos(0), - _limit(0), - _bufHolder(NULL), _ownedBuffer() { - set(buffer, len); } -ByteBuffer::ByteBuffer(Alloc buffer, size_t len) : +ByteBuffer::ByteBuffer(Alloc buffer, uint32_t len) : _buffer(static_cast<char *>(buffer.get())), _len(len), _pos(0), - _limit(len), - _bufHolder(NULL), _ownedBuffer(std::move(buffer)) { } -ByteBuffer::ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit) : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(buf, pos, len, limit); - LOG_DEBUG3("Created copy of byte buffer of length %" PRIu64 " with " - "limit %" PRIu64 ".", len, limit); -} - -ByteBuffer::ByteBuffer(const ByteBuffer& bb) : - _buffer(0), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), +ByteBuffer::ByteBuffer(const ByteBuffer& rhs) : + _buffer(nullptr), + _len(rhs._len), + _pos(rhs._pos), _ownedBuffer() { - LOG_DEBUG1("Created empty byte buffer to assign to."); - *this = bb; -} - -ByteBuffer& ByteBuffer::operator=(const ByteBuffer & org) -{ - if (this != & org) { - cleanUp(); - if (org._len > 0 && org._buffer) { - Alloc::alloc(org._len + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer,org._buffer,org._len); - _buffer[org._len] = 0; - } - _len = org._len; - _pos = org._pos; - _limit = org._limit; - LOG_DEBUG4("Assignment created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); - } - return *this; -} - -void -ByteBuffer::set(BufferHolder* buf, size_t pos, size_t len, size_t limit) -{ - cleanUp(); - _bufHolder = buf; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=pos; - _len=len; - _limit=limit; - LOG_DEBUG4("set() created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); -} - -ByteBuffer::~ByteBuffer() -{ - if (_bufHolder) { - _bufHolder->subRef(); + if (rhs._len > 0 && rhs._buffer) { + Alloc buf = Alloc::alloc(rhs._len); + memcpy(buf.get(), rhs._buffer, rhs._len); + _ownedBuffer = std::move(buf); + _buffer = static_cast<const char *>(_ownedBuffer.get()); } } -std::unique_ptr<ByteBuffer> -ByteBuffer::sliceCopy() const -{ - ByteBuffer* buf = new ByteBuffer; - buf->sliceFrom(*this, _pos, _limit); - - LOG_DEBUG3("Created slice at pos %" PRIu64 " with limit %" PRIu64 ".", - _pos, _limit); - return std::unique_ptr<ByteBuffer>(buf); -} +ByteBuffer::~ByteBuffer() = default; -void ByteBuffer::throwOutOfBounds(size_t want, size_t has) -{ - LOG_DEBUG1("Throwing out of bounds exception"); - throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); -} - -void -ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Created slice from buffer from %" PRIu64 " to %" PRIu64 ".", - from, to); - if (from > buf._len) { - throwOutOfBounds(from, buf._len); - } else if (to > buf._len) { - throwOutOfBounds(to, buf._len); - } else if (to < from) { - throwOutOfBounds(to, from); - } else { - - if (!buf._buffer) { - clear(); - return; - } - - // Slicing from someone that doesn't own their buffer, must make own copy. - if (( buf._ownedBuffer.get() == NULL ) && (buf._bufHolder == NULL)) { - cleanUp(); - Alloc::alloc(to-from + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer, buf._buffer + from, to-from); - _buffer[to-from] = 0; - _pos = 0; - _len = _limit = to-from; - return; - } - - // Slicing from someone that owns, but hasn't made a reference counter yet. - if (!buf._bufHolder) { - buf._bufHolder=new BufferHolder(std::move(const_cast<Alloc &>(buf._ownedBuffer))); - } - - // Slicing from refcounter. - cleanUp(); - - _bufHolder = buf._bufHolder; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=from; - _len=to; - _limit=to; - } -} - -ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, size_t len) +ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, uint32_t len) { if (buffer && len) { - Alloc newBuf = Alloc::alloc(len + 1); + Alloc newBuf = Alloc::alloc(len); memcpy(newBuf.get(), buffer, len); - static_cast<char *>(newBuf.get())[len] = 0; return new ByteBuffer(std::move(newBuf), len); } else { - return NULL; - } -} - -void -ByteBuffer::setPos(size_t pos) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting pos to be %" PRIu64 ", limit is %" PRIu64 ".", - pos, _limit); - if (pos>_limit) { - throwOutOfBounds(pos, _limit); - } else { - _pos=pos; - } -} - -void -ByteBuffer::setLimit(size_t limit) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting limit to %" PRIu64 ", (size is %" PRIu64 ").", limit, _len); - if (limit>_len) { - throwOutOfBounds(limit, _len); - } else { - _limit=limit; + return nullptr; } } - -ByteBuffer::BufferHolder::BufferHolder(Alloc buffer) - : _buffer(std::move(buffer)) -{ -} - -ByteBuffer::BufferHolder::~BufferHolder() = default; -void ByteBuffer::dump() const +void ByteBuffer::incPos(uint32_t pos) { - fprintf(stderr, "ByteBuffer: Length %lu, Pos %lu, Limit %lu\n", - _len, _pos, _limit); - for (size_t i=0; i<_len; i++) { - if (_buffer[i]>32 && _buffer[i]<126) { - fprintf(stderr, "%c", _buffer[i]); - } else { - fprintf(stderr, "[%d]",_buffer[i]); - } - } -} - -void ByteBuffer::incPos(size_t pos) -{ - LOG_DEBUG2("incPos(%" PRIu64 ")", pos); - if (_pos + pos > _limit) { - throwOutOfBounds(_pos + pos, _limit); + if (_pos + pos > _len) { + throwOutOfBounds(_pos + pos, _len); } else { _pos+=pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif } } void ByteBuffer::getNumeric(uint8_t & v) { - LOG_DEBUG2("getNumeric8(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(uint8_t *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(uint8_t v) { - LOG_DEBUG2("putNumeric8(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - *(uint8_t *) getBufferAtPos() = v; + v = *reinterpret_cast<const uint8_t *>(getBufferAtPos()); incPosNoCheck(sizeof(v)); } } -size_t ByteBuffer::forceValgrindStart2Pos() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(_buffer), *e(c + _pos); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - -size_t ByteBuffer::forceValgrindPos2Lim() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(getBufferAtPos()), *e(c + getRemaining()); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - - void ByteBuffer::getNumericNetwork(int16_t & v) { - LOG_DEBUG2("getNumericNetwork16(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint16_t val = *(uint16_t *) (void *) getBufferAtPos(); + uint16_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohs(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int16_t & v) { - LOG_DEBUG2("getNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int16_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(int16_t v) { - LOG_DEBUG2("putNumericNetwork16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint16_t val = htons(v); - *(uint16_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int16_t v) { - LOG_DEBUG2("putNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int16_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - void ByteBuffer::getNumericNetwork(int32_t & v) { - LOG_DEBUG2("getNumericNetwork32(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint32_t val = *(uint32_t *) (void *) getBufferAtPos(); + uint32_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohl(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int32_t & v) { - LOG_DEBUG2("getNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int32_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - - -void ByteBuffer::putNumericNetwork(int32_t v) { - LOG_DEBUG2("putNumericNetwork32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint32_t val = htonl(v); - *(uint32_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int32_t v) { - LOG_DEBUG2("putNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int32_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::getNumericNetwork(float & v) { - LOG_DEBUG2("getNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(uint32_t) == 4 - // and endianness same for float and ints - int32_t val; - getIntNetwork(val); - memcpy(&v, &val, sizeof(v)); -} - -void ByteBuffer::getNumeric(float & v) { - LOG_DEBUG2("getNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(float *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(float v) { - LOG_DEBUG2("putNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(int32_t) == 4 - // and endianness same for float and ints - int32_t val; - memcpy(&val, &v, sizeof(val)); - putIntNetwork(val); -} - -void ByteBuffer::putNumeric(float v) { - LOG_DEBUG2("putNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(float *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} void ByteBuffer::getNumeric(int64_t& v) { - LOG_DEBUG2("getNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int64_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(int64_t v) { - LOG_DEBUG2("putNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int64_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::getNumeric(double& v) { - LOG_DEBUG2("getNumericDouble(%f)", v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - v = *(double *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(double v) { - LOG_DEBUG2("putNumericDouble(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(double *) (void *) getBufferAtPos() = v; + memcpy(&v, getBufferAtPos(), sizeof(v)); incPosNoCheck(sizeof(v)); } } void ByteBuffer::getNumericNetwork(double & v) { - LOG_DEBUG2("getNumericNetworkDouble(%f)", v); getDoubleLongNetwork(v); } -void ByteBuffer::putNumericNetwork(int64_t v) { - LOG_DEBUG2("putNumericNetwork64(%" PRId64 ")", v); - putDoubleLongNetwork(v); -} -void ByteBuffer::putNumericNetwork(double v) { - LOG_DEBUG2("putNumericNetworkDouble(%f)", v); - putDoubleLongNetwork(v); -} + void ByteBuffer::getNumericNetwork(int64_t & v) { - LOG_DEBUG2("getNumericNetwork64(%" PRId64 ")", v); getDoubleLongNetwork(v); } -void ByteBuffer::putInt2_4_8Bytes(int64_t number, size_t len) { - LOG_DEBUG3("putInt2_4_8(%" PRId64 ", %" PRIu64 ")", number, len); - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (len == 0) { - if (number < 0x8000ll) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (number < 0x40000000ll) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } - } else if (len == 2) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (len == 4) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else if (len == 8) { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } else { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number using %d bytes.", (int)len), VESPA_STRLOC); - } -} - -void ByteBuffer::getInt2_4_8Bytes(int64_t & v) { - LOG_DEBUG2("getInt2_4_8(%" PRId64 ")", v); - if (getRemaining() >= 2) { - uint8_t flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 8 bytes - int64_t tmp; - getLongNetwork(tmp); - v = tmp & 0x3FFFFFFFFFFFFFFFll; - } else { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = (int64_t) (tmp & 0x3FFFFFFF); - } - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int64_t) tmp; - } - } else { - throwOutOfBounds(getRemaining(), 2); - } -} - -size_t ByteBuffer::getSerializedSize2_4_8Bytes(int64_t number) { - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (number < 0x8000ll) { - return 2; - } else if (number < 0x40000000ll) { - return 4; - } else { - return 8; - } -} - -void ByteBuffer::putInt1_2_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_2_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else if (number < 0x4000) { - putShortNetwork((int16_t) (((int16_t)number) | ((int16_t) 0x8000))); - } else { - putIntNetwork(number | 0xC0000000); - } -} - -void ByteBuffer::getInt1_2_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_2_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x3FFFFFFF; - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int32_t) (tmp & ((int16_t) 0x3FFF)); - } - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} - -size_t ByteBuffer::getSerializedSize1_2_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else if (number < 0x4000) { - return 2; - } else { - return 4; - } -} -void ByteBuffer::putInt1_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else { - putIntNetwork(number | 0x80000000); - } -} -void ByteBuffer::getInt1_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x7FFFFFFF; - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} -size_t ByteBuffer::getSerializedSize1_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else { - return 4; - } -} -void ByteBuffer::getBytes(void *buffer, size_t count) +void ByteBuffer::getBytes(void *buffer, uint32_t count) { - LOG_DEBUG3("getBytes(%p, %" PRIu64 ")", buffer, count); const char *v = getBufferAtPos(); incPos(count); memcpy(buffer, v, count); } -void ByteBuffer::putBytes(const void *buf, size_t count) { - LOG_DEBUG3("putBytes(%p, %" PRIu64 ")", buf, count); - if (__builtin_expect(getRemaining() < count, 0)) { - throwOutOfBounds(getRemaining(), sizeof(count)); - } else { - memcpy(getBufferAtPos(), buf, count); - incPosNoCheck(count); - } -} -std::string ByteBuffer::toString() { - std::ostringstream ost; - StringUtil::printAsHex(ost, getBuffer(), getLength()); - return ost.str(); -} - -void ByteBuffer::swap(ByteBuffer& other) { - LOG_DEBUG2("swap(%p)", &other); - std::swap(_bufHolder, other._bufHolder); - std::swap(_buffer, other._buffer); - std::swap(_len, other._len); - std::swap(_pos, other._pos); - std::swap(_limit, other._limit); -} - -void ByteBuffer::cleanUp() { - LOG_DEBUG1("cleanUp()"); - if (_bufHolder) { - _bufHolder->subRef(); - _bufHolder = NULL; - } else { - Alloc().swap(_ownedBuffer); - } - _buffer = NULL; -} } // document diff --git a/document/src/vespa/document/util/bytebuffer.h b/document/src/vespa/document/util/bytebuffer.h index 6467e6d8bf0..17ab0322a7d 100644 --- a/document/src/vespa/document/util/bytebuffer.h +++ b/document/src/vespa/document/util/bytebuffer.h @@ -15,7 +15,6 @@ #pragma once #include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/referencecounter.h> namespace document { @@ -23,27 +22,21 @@ class ByteBuffer { public: typedef std::unique_ptr<ByteBuffer> UP; - /** - * Creates a byte buffer with no underlying buffer. - * Use set() to set the buffer. - */ - ByteBuffer(); ByteBuffer(const ByteBuffer &); - ByteBuffer& operator=(const ByteBuffer &); + ByteBuffer& operator=(const ByteBuffer &) = delete; + ByteBuffer(ByteBuffer &&) = default; + ByteBuffer& operator=(ByteBuffer &&) = delete; ~ByteBuffer(); - /** Allocates buffer with len bytes. */ - ByteBuffer(size_t len); - /** * Create a buffer with the given content. * * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(const char* buffer, size_t len); + ByteBuffer(const char* buffer, uint32_t len); /** * Create a buffer with the given content. @@ -51,22 +44,7 @@ public: * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(vespalib::alloc::Alloc buffer, size_t len); - - /** - * Sets the buffer pointed to by this buffer. Allows for multiple - * usages of the same ByteBuffer. - */ - void set(const char* buffer, size_t len) { - cleanUp(); - _buffer = const_cast<char*>(buffer); - _len=len; - _limit=len; - _pos=0; - } - - /** Clear this buffer, and set free the underlying BufferHolder. */ - void reset() { set(NULL, 0); } + ByteBuffer(vespalib::alloc::Alloc buffer, uint32_t len); /** * Creates a ByteBuffer object from another buffer. allocates @@ -75,62 +53,28 @@ public: * @param buffer The buffer to copy. * @param len The length of the buffer. * - * @return Returns a newly created bytebuffer object, or NULL - * if buffer was NULL, or len was <=0. + * @return Returns a newly created bytebuffer object, or nullptr + * if buffer was nullptr, or len was <=0. */ - static ByteBuffer* copyBuffer(const char* buffer, size_t len); - - std::unique_ptr<ByteBuffer> sliceCopy() const; - - /** - * @throws BufferOutOfBoundsException If faulty range is given. - */ - void sliceFrom(const ByteBuffer& buf, size_t from, size_t to); + static ByteBuffer* copyBuffer(const char* buffer, uint32_t len); /** @return Returns the buffer pointed to by this object (at position 0) */ - char* getBuffer() const { return _buffer; } + const char* getBuffer() const { return _buffer; } /** @return Returns the length of the buffer pointed to by this object. */ - size_t getLength() const { return _len; } - - /** - * Adjust the length of the buffer. Only sane to shorten it, as you do not - * know what is ahead. - */ - void setLength(size_t len) { _len = len; } + uint32_t getLength() const { return _len; } /** @return Returns a pointer to the current position in the buffer. */ - char* getBufferAtPos() const { return _buffer + _pos; } + const char* getBufferAtPos() const { return _buffer + _pos; } /** @return Returns the index of the current position in the buffer. */ - size_t getPos() const { return _pos; } - - /** @return Returns the limit. */ - size_t getLimit() const { return _limit; } + uint32_t getPos() const { return _pos; } /** * @return Returns the number of bytes remaining in the buffer - that is, - * getLimit()-getPos(). + * getLength()-getPos(). */ - size_t getRemaining() const { return _limit-_pos; } - - /** - * Changes the position in the buffer. - * - * @throws BufferOutOfBoundsException; - */ - void setPos(size_t pos); - - /** - * Sets the buffer limit. - * - * @param limit The new limit. - * @return True if the limit is legal (less than the length) - * @throws BufferOutOfBoundsException; - */ - void setLimit(size_t limit); - size_t forceValgrindStart2Pos() const __attribute__ ((noinline)); - size_t forceValgrindPos2Lim() const __attribute__ ((noinline)); + uint32_t getRemaining() const { return _len -_pos; } /** * Moves the position in the buffer. @@ -138,267 +82,37 @@ public: * @param pos The number of bytes to move the position. The new position * will be oldPos + pos. This is the same as doing * setPos(getPos()+pos) - * @return True if the position could be moved (it was inside the limit - * of the buffer). * @throws BufferOutOfBoundsException; */ - void incPos(size_t pos); - - void incPosNoCheck(size_t pos) { - _pos += pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif - } - - /** - * Resets pos to 0, and sets limit to old pos. Use this before reading - * from a buffer you have written to - */ - void flip() { - _limit = _pos; - _pos = 0; - } + void incPos(uint32_t pos); - /** - * Sets pos to 0 and limit to length. Use this to start writing from the - * start of the buffer. - */ - void clear() { - _pos=0; - _limit=_len; - } - - void getNumericNetwork(uint8_t & v) { getNumeric(v); } void getNumeric(uint8_t & v); - void putNumericNetwork(uint8_t v) { putNumeric(v); } - void putNumeric(uint8_t v); void getNumericNetwork(int16_t & v); - void getNumeric(int16_t & v); - void putNumericNetwork(int16_t v); - void putNumeric(int16_t v); void getNumericNetwork(int32_t & v); - void getNumeric(int32_t & v); - void putNumericNetwork(int32_t v); - void putNumeric(int32_t v); - void getNumericNetwork(float & v); - void getNumeric(float & v); - void putNumericNetwork(float v); - void putNumeric(float v); + void getNumericNetwork(int64_t & v); void getNumeric(int64_t& v); - void putNumericNetwork(int64_t v); - void putNumeric(int64_t v); void getNumericNetwork(double & v); - void getNumeric(double& v); - void putNumericNetwork(double v); - void putNumeric(double v); + void getChar(char & val) { unsigned char t;getByte(t); val=t; } void getByte(uint8_t & v) { getNumeric(v); } - void putByte(uint8_t v) { putNumeric(v); } void getShortNetwork(int16_t & v) { getNumericNetwork(v); } - void getShort(int16_t & v) { getNumeric(v); } - void putShortNetwork(int16_t v) { putNumericNetwork(v); } - void putShort(int16_t v) { putNumeric(v); } void getIntNetwork(int32_t & v) { getNumericNetwork(v); } - void getInt(int32_t & v) { getNumeric(v); } - void putIntNetwork(int32_t v) { putNumericNetwork(v); } - void putInt(int32_t v) { putNumeric(v); } - void getFloatNetwork(float & v) { getNumericNetwork(v); } - void getFloat(float & v) { getNumeric(v); } - void putFloatNetwork(float v) { putNumericNetwork(v); } - void putFloat(float v) { putNumeric(v); } void getLongNetwork(int64_t & v) { getNumericNetwork(v); } void getLong(int64_t& v) { getNumeric(v); } - void putLongNetwork(int64_t v) { putNumericNetwork(v); } - void putLong(int64_t v) { putNumeric(v); } void getDoubleNetwork(double & v) { getNumericNetwork(v); } - void getDouble(double& v) { getNumeric(v); } - void putDoubleNetwork(double v) { putNumericNetwork(v); } - void putDouble(double v) { putNumeric(v); } - - private: - void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline,noreturn)); - uint8_t peekByte() const { return *getBufferAtPos(); } - -#if defined(__i386__) || defined(__x86_64__) - - template<typename T> - void putDoubleLongNetwork(T val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - putByte(data[i]); - } - } + void getBytes(void *buffer, uint32_t count); +private: template<typename T> - void getDoubleLongNetwork(T &val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } + void getDoubleLongNetwork(T &val); - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - getByte(data[i]); - } - } -#else - #error "getDoubleLongNetwork is undefined for this arcitecture" -#endif + void incPosNoCheck(uint32_t pos) { _pos += pos; } - public: - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - */ - void putInt2_4_8Bytes(int64_t number) { - putInt2_4_8Bytes(number, 0); - } - - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - * @param len if non-zero, force writing number using len bytes, possibly - * with truncation - */ - void putInt2_4_8Bytes(int64_t number, size_t len); - - /** - * Reads a 62-bit positive integer from the buffer, which was written using - * 2, 4, or 8 bytes. - * - * @param v the integer read - */ - void getInt2_4_8Bytes(int64_t & v); - - /** - * Computes the size used for storing the given integer using 2, 4 or 8 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 2, 4 or 8 - */ - static size_t getSerializedSize2_4_8Bytes(int64_t number); - - /** - * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_2_4Bytes(int32_t number); - - /** - * Reads a 30-bit positive integer from the buffer, which was written using - * 1, 2, or 4 bytes. - * - * @param v the integer read - */ - void getInt1_2_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1, 2 or 4 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1, 2 or 4 - */ - static size_t getSerializedSize1_2_4Bytes(int32_t number); - - /** - * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_4Bytes(int32_t number); - - /** - * Reads a 31-bit positive integer from the buffer, which was written using - * 1 or 4 bytes. - * - * @param v the integer read - */ - void getInt1_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1 or 4 bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1 or 4 - */ - static size_t getSerializedSize1_4Bytes(int32_t number); - - /** - * Writes a 8 bit integer to the buffer at the current position, and - * increases the positition accordingly. - * - * @param val the int to store - * @return True if the value could be stored, false if end of buffer is - * reached - */ - void getChar(char & val) { unsigned char t;getByte(t); val=t; } - void putChar(char val) { putByte(static_cast<unsigned char>(val)); } - - /** - * Reads the given number of bytes into the given pointer, and updates the - * positition accordingly - * - * @param buffer where to store the bytes - * @param count number of bytes to read - * @return True if all the bytes could be read, false if end of - * buffer is reached - */ - void getBytes(void *buffer, size_t count); - - /** - * Writes the given number of bytes into the ByteBuffer at the current - * position, and updates the positition accordingly - * - * @param buf the bytes to store - * @param count number of bytes to store - */ - void putBytes(const void *buf, size_t count); - - /** Debug */ - void dump() const; - - class BufferHolder : public vespalib::ReferenceCounter - { - private: - BufferHolder(const BufferHolder &); - BufferHolder& operator=(const BufferHolder &); - - public: - BufferHolder(vespalib::alloc::Alloc buffer); - virtual ~BufferHolder(); - - vespalib::alloc::Alloc _buffer; - }; - - ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit); - - void set(BufferHolder* buf, size_t pos, size_t len, size_t limit); - -private: - char * _buffer; - size_t _len; - size_t _pos; - size_t _limit; - mutable BufferHolder * _bufHolder; + const char * _buffer; + const uint32_t _len; + uint32_t _pos; vespalib::alloc::Alloc _ownedBuffer; -public: - - std::string toString(); - - void swap(ByteBuffer& other); - - void cleanUp(); }; } // document diff --git a/document/src/vespa/document/util/serializable.cpp b/document/src/vespa/document/util/serializable.cpp deleted file mode 100644 index 32c7bef90f0..00000000000 --- a/document/src/vespa/document/util/serializable.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "serializable.h" -#include "bufferexceptions.h" -#include "serializableexceptions.h" -#include "bytebuffer.h" - -namespace document { - -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Serializable, vespalib::Identifiable); -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Deserializable, Serializable); -VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(SerializeException); - -std::unique_ptr<ByteBuffer> Serializable::serialize() const -{ - size_t len = getSerializedSize(); - std::unique_ptr<ByteBuffer> retVal(new ByteBuffer(len)); - serialize(*retVal.get()); - return retVal; -} - -DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -DeserializeException::DeserializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -SerializeException::SerializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -SerializeException::SerializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -void -Serializable::serialize(ByteBuffer& buffer) const { - int pos = buffer.getPos(); - try{ - onSerialize(buffer); - } catch (...) { - buffer.setPos(pos); - throw; - } -} - -void -Deserializable::deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) { - int pos = buffer.getPos(); - try { - onDeserialize(repo, buffer); - } catch (const DeserializeException &) { - buffer.setPos(pos); - throw; - } catch (const BufferOutOfBoundsException &) { - buffer.setPos(pos); - throw; - } -} -} diff --git a/document/src/vespa/document/util/serializable.h b/document/src/vespa/document/util/serializable.h deleted file mode 100644 index 8818feceb9a..00000000000 --- a/document/src/vespa/document/util/serializable.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file serializable.h - * @ingroup document - * - * @brief Interfaces to be used for serializing of objects. - * - * @author Thomas F. Gundersen, H�kon Humberset - * @date 2004-03-15 - * @version $Id$ - */ - -#pragma once - -#include <vespa/vespalib/objects/cloneable.h> -#include <vespa/vespalib/objects/identifiable.h> - -#include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/identifiableid.h> - -namespace document { -class DocumentTypeRepo; - -/** - * Base class for classes that can be converted into a bytestream, - * normally used later to create a similar instance. - */ - -class Serializable : public vespalib::Identifiable -{ -protected: - virtual void onSerialize(ByteBuffer& buffer) const = 0; -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Serializable); - - virtual ~Serializable() {} - - /** - * @return An upper limit to how many bytes serialization of this instance - * need, providing instance is not altered before serialization. - */ - virtual size_t getSerializedSize() const = 0; - - /** - * Serializes the instance into the buffer given. Use getSerializedSize() - * before calling this method to be sure buffer is big enough. - * On success, the given buffers position will be just past the serialized - * version of this instance, on failure, position will be reset to whatever - * it was prior to calling this function. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - void serialize(ByteBuffer& buffer) const; - - /** - * Creates a bytebuffer with enough space to serialize this instance - * and serialize this instance into it. - * - * @return The created bytebuffer, positioned after the serialization. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - std::unique_ptr<ByteBuffer> serialize() const; -}; - -/** - * Base class for instances that can be overwritten from a bytestream, - * given that the bytestream is created from a similar instance. - */ -class Deserializable : public vespalib::Cloneable, public Serializable -{ -protected: - virtual void onDeserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) = 0; - -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Deserializable); - virtual ~Deserializable() {} - - /** - * Overwrite this object with the object represented by the given - * bytestream. On success, buffer will be positioned after the bytestream - * representing the instance we've just deserialized, on failure, bytebuffer - * will be pointing to where it was pointing before calling this function. - * - * @throw DeserializeException If read data doesn't represent a legal object - * of this type. - * @throw BufferOutOfBoundsException If instance wants to read more data - * than is available in the buffer. - */ - void deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer); -}; - -} - diff --git a/document/src/vespa/document/util/serializableexceptions.cpp b/document/src/vespa/document/util/serializableexceptions.cpp new file mode 100644 index 00000000000..e80e38015e8 --- /dev/null +++ b/document/src/vespa/document/util/serializableexceptions.cpp @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serializableexceptions.h" + +namespace document { + +VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); + +DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, location, 1) +{ +} + +DeserializeException::DeserializeException( + const vespalib::string& msg, const vespalib::Exception& cause, + const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) +{ +} + +} diff --git a/document/src/vespa/document/util/serializableexceptions.h b/document/src/vespa/document/util/serializableexceptions.h index fcfed810bfc..1b692aa27b7 100644 --- a/document/src/vespa/document/util/serializableexceptions.h +++ b/document/src/vespa/document/util/serializableexceptions.h @@ -24,12 +24,4 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(DeserializeException) }; -class SerializeException : public vespalib::IoException { -public: - SerializeException(const vespalib::string& msg, const vespalib::string& location = ""); - SerializeException(const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location = ""); - VESPA_DEFINE_EXCEPTION_SPINE(SerializeException) -}; - } diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 79f7d7c0ccc..5582c0ea153 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -20,7 +20,6 @@ namespace documentapi { class LoadTypeSet; class RoutingPolicyRepository; class RoutableRepository; -class SystemState; class IRoutingPolicyFactory; class IRoutableFactory; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp index 718611dfc04..6c8394b1b4c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp @@ -15,12 +15,11 @@ DocumentState::DocumentState(const DocumentState& o) : _gid(o._gid), _timestamp(o._timestamp), _removeEntry(o._removeEntry) { if (o._docId.get() != 0) { - _docId.reset(new document::DocumentId(*o._docId)); + _docId = std::make_unique<document::DocumentId>(*o._docId); } } -DocumentState::DocumentState(const document::DocumentId& id, - uint64_t timestamp, bool removeEntry) +DocumentState::DocumentState(const document::DocumentId& id, uint64_t timestamp, bool removeEntry) : _docId(new document::DocumentId(id)), _gid(_docId->getGlobalId()), _timestamp(timestamp), @@ -28,8 +27,7 @@ DocumentState::DocumentState(const document::DocumentId& id, { } -DocumentState::DocumentState(const document::GlobalId& gid, - uint64_t timestamp, bool removeEntry) +DocumentState::DocumentState(const document::GlobalId& gid, uint64_t timestamp, bool removeEntry) : _gid(gid), _timestamp(timestamp), _removeEntry(removeEntry) {} DocumentState::DocumentState(document::ByteBuffer& buf) @@ -39,10 +37,10 @@ DocumentState::DocumentState(document::ByteBuffer& buf) buf.getByte(hasDocId); if (hasDocId) { vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); - _docId.reset(new document::DocumentId(stream)); + _docId = std::make_unique<document::DocumentId>(stream); buf.incPos(stream.rp()); } - char* gid = buf.getBufferAtPos(); + const char* gid = buf.getBufferAtPos(); buf.incPos(document::GlobalId::LENGTH); _gid.set(gid); buf.getLongNetwork((int64_t&) _timestamp); @@ -55,8 +53,8 @@ DocumentState& DocumentState::operator=(const DocumentState& other) { _docId.reset(); - if (other._docId.get() != 0) { - _docId.reset(new document::DocumentId(*other._docId)); + if (other._docId) { + _docId = std::make_unique<document::DocumentId>(*other._docId); } _gid = other._gid; _timestamp = other._timestamp; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp index 6753d269ad6..2dcaccbd861 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp @@ -21,12 +21,12 @@ PutDocumentMessage::PutDocumentMessage(document::Document::SP document) : setDocument(std::move(document)); } -PutDocumentMessage::~PutDocumentMessage() {} +PutDocumentMessage::~PutDocumentMessage() = default; DocumentReply::UP PutDocumentMessage::doCreateReply() const { - return DocumentReply::UP(new WriteDocumentReply(DocumentProtocol::REPLY_PUTDOCUMENT)); + return std::make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); } bool diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp index 411a7237cb5..08d631dc44b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp @@ -4,11 +4,7 @@ namespace documentapi { -QueryResultMessage::QueryResultMessage() : - VisitorMessage(), - _searchResult(), - _summary() -{} +QueryResultMessage::QueryResultMessage() = default; QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary) : VisitorMessage(), @@ -16,7 +12,7 @@ QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, cons _summary(summary) {} -QueryResultMessage::~QueryResultMessage() {} +QueryResultMessage::~QueryResultMessage() = default; DocumentReply::UP QueryResultMessage::doCreateReply() const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h index 239ce6fefd5..6324e2664e4 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h @@ -25,7 +25,7 @@ public: * Constructs a new search result message for deserialization. */ QueryResultMessage(); - ~QueryResultMessage(); + ~QueryResultMessage() override; /** * Constructs a new search result message for the given search result. diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp index 43ad30ea24f..453e93fd7eb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -1,8 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visitor.h" -#include <climits> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> +#include <climits> using document::FixedBucketSpaces; @@ -56,7 +59,7 @@ CreateVisitorMessage::~CreateVisitorMessage() = default; DocumentReply::UP CreateVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); + return std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); } uint32_t @@ -65,11 +68,7 @@ CreateVisitorMessage::getType() const return DocumentProtocol::MESSAGE_CREATEVISITOR; } -DestroyVisitorMessage::DestroyVisitorMessage() : - DocumentMessage(), - _instanceId() -{ -} +DestroyVisitorMessage::DestroyVisitorMessage() = default; DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : DocumentMessage(), @@ -77,13 +76,12 @@ DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : { } -DestroyVisitorMessage::~DestroyVisitorMessage() { -} +DestroyVisitorMessage::~DestroyVisitorMessage() = default; DocumentReply::UP DestroyVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_DESTROYVISITOR)); + return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_DESTROYVISITOR); } uint32_t @@ -104,20 +102,13 @@ CreateVisitorReply::CreateVisitorReply(uint32_t type) : { } -VisitorInfoMessage::VisitorInfoMessage() : - VisitorMessage(), - _finishedBuckets(), - _errorMessage() -{ -} - -VisitorInfoMessage::~VisitorInfoMessage() { -} +VisitorInfoMessage::VisitorInfoMessage() = default; +VisitorInfoMessage::~VisitorInfoMessage() = default; DocumentReply::UP VisitorInfoMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); } uint32_t @@ -126,11 +117,7 @@ VisitorInfoMessage::getType() const return DocumentProtocol::MESSAGE_VISITORINFO; } -MapVisitorMessage::MapVisitorMessage() : - _data() -{ - // empty -} +MapVisitorMessage::MapVisitorMessage() = default; uint32_t MapVisitorMessage::getApproxSize() const @@ -141,7 +128,7 @@ MapVisitorMessage::getApproxSize() const DocumentReply::UP MapVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); } uint32_t MapVisitorMessage::getType() const @@ -149,60 +136,37 @@ uint32_t MapVisitorMessage::getType() const return DocumentProtocol::MESSAGE_MAPVISITOR; } -DocumentListMessage::Entry::Entry() -{ - // empty -} +DocumentListMessage::Entry::Entry() = default; -DocumentListMessage::Entry::Entry(int64_t timestamp, - document::Document::SP doc, - bool removeEntry) : +DocumentListMessage::Entry::Entry(int64_t timestamp, document::Document::SP doc, bool removeEntry) : _timestamp(timestamp), - _document(doc), + _document(std::move(doc)), _removeEntry(removeEntry) -{ - // empty -} +{ } -DocumentListMessage::Entry::Entry(const Entry& other) : - _timestamp(other._timestamp), - _document(other._document), - _removeEntry(other._removeEntry) -{ - // empty -} +DocumentListMessage::Entry::Entry(const Entry& other) = default; -DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buf) +DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, document::ByteBuffer& buf) { buf.getLongNetwork(_timestamp); - _document.reset(new document::Document(repo, buf)); + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + _document = std::make_unique<document::Document>(repo, stream); + buf.incPos(buf.getRemaining() - stream.size()); uint8_t b; buf.getByte(b); _removeEntry = b>0; } void -DocumentListMessage::Entry::serialize(document::ByteBuffer& buf) const +DocumentListMessage::Entry::serialize(vespalib::GrowableByteBuffer& buf) const { - buf.putLongNetwork(_timestamp); - _document->serialize(buf); + buf.putLong(_timestamp); + vespalib::nbostream nbo = _document->serialize(); + buf.putBytes(nbo.data(), nbo.size()); buf.putByte(_removeEntry ? 1 : 0); } -uint32_t -DocumentListMessage::Entry::getSerializedSize() const -{ - return sizeof(int64_t) + sizeof(uint8_t) - + _document->serialize()->getLength(); -} - -DocumentListMessage::DocumentListMessage() : - _bucketId(), - _documents() -{ - // empty -} +DocumentListMessage::DocumentListMessage() = default; DocumentListMessage::DocumentListMessage(document::BucketId bid) : _bucketId(bid), @@ -214,7 +178,7 @@ DocumentListMessage::DocumentListMessage(document::BucketId bid) : DocumentReply::UP DocumentListMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); } uint32_t diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index b18a4e985f3..f47fa48bd80 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -52,7 +52,7 @@ public: const string& instanceId, const string& controlDestination, const string& dataDestination); - ~CreateVisitorMessage(); + ~CreateVisitorMessage() override; const string& getLibraryName() const { return _libName; } void setLibraryName(const string& value) { _libName = value; } @@ -250,7 +250,7 @@ public: const document::Document::SP& getDocument() { return _document; } bool isRemoveEntry() { return _removeEntry; } - void serialize(document::ByteBuffer& buf) const; + void serialize(vespalib::GrowableByteBuffer& buf) const; uint32_t getSerializedSize() const; private: int64_t _timestamp; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 797f55120fc..7bae7dd7e77 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -20,7 +20,7 @@ namespace documentapi { bool RoutableFactories60::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const { - const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); + const auto &msg = static_cast<const DocumentMessage&>(obj); out.putByte(msg.getPriority()); out.putInt(msg.getLoadType().getId()); return doEncode(msg, out); @@ -93,7 +93,7 @@ RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer msg->setVisitRemoves(decodeBoolean(buf)); msg->setFieldSet(decodeString(buf)); msg->setVisitInconsistentBuckets(decodeBoolean(buf)); - msg->getParameters().deserialize(_repo, buf); + msg->getParameters().deserialize(buf); msg->setVisitorDispatcherVersion(50); decodeInt(buf); // Unused legacy visitor ordering msg->setMaxBucketsPerVisitor(decodeInt(buf)); @@ -105,7 +105,7 @@ RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); + const auto &msg = static_cast<const CreateVisitorMessage&>(obj); buf.putString(msg.getLibraryName()); buf.putString(msg.getInstanceId()); @@ -126,10 +126,8 @@ RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage buf.putString(msg.getFieldSet()); buf.putBoolean(msg.visitInconsistentBuckets()); - int len = msg.getParameters().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getParameters().serialize(dbuf); + + msg.getParameters().serialize(buf); buf.putInt(0); // Unused legacy visitor ordering buf.putInt(msg.getMaxBucketsPerVisitor()); @@ -184,7 +182,7 @@ RoutableFactories60::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::CreateVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const CreateVisitorReply &reply = static_cast<const CreateVisitorReply&>(obj); + const auto &reply = static_cast<const CreateVisitorReply&>(obj); buf.putLong(reply.getLastBucket().getRawId()); buf.putInt(reply.getVisitorStatistics().getBucketsVisited()); buf.putLong(reply.getVisitorStatistics().getDocumentsVisited()); @@ -209,19 +207,14 @@ RoutableFactories60::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, } DocumentReply::UP -RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const +RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& ) const { - (void) buf; - return DocumentReply::UP(new DocumentIgnoredReply()); + return std::make_unique<DocumentIgnoredReply>(); } bool -RoutableFactories60::DocumentIgnoredReplyFactory::doEncode( - const DocumentReply& obj, - vespalib::GrowableByteBuffer& buf) const +RoutableFactories60::DocumentIgnoredReplyFactory::doEncode(const DocumentReply&, vespalib::GrowableByteBuffer& ) const { - (void) obj; - (void) buf; return true; } @@ -243,15 +236,12 @@ RoutableFactories60::DocumentListMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::DocumentListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const DocumentListMessage &msg = static_cast<const DocumentListMessage&>(obj); + const auto &msg = static_cast<const DocumentListMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); buf.putInt(msg.getDocuments().size()); for (const auto & document : msg.getDocuments()) { - int len = document.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - document.serialize(dbuf); + document.serialize(buf); } return true; @@ -283,11 +273,7 @@ bool RoutableFactories60::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); - - int32_t len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); + msg.serialize(buf); return true; } @@ -322,7 +308,7 @@ RoutableFactories60::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::EmptyBucketsMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); + const auto &msg = static_cast<const EmptyBucketsMessage&>(obj); buf.putInt(msg.getBucketIds().size()); for (const auto & bucketId : msg.getBucketIds()) { @@ -367,7 +353,7 @@ RoutableFactories60::GetBucketListMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); + const auto &msg = static_cast<const GetBucketListMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); return encodeBucketSpace(msg.getBucketSpace(), buf); } @@ -392,7 +378,7 @@ RoutableFactories60::GetBucketListReplyFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::GetBucketListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketListReply &reply = static_cast<const GetBucketListReply&>(obj); + const auto &reply = static_cast<const GetBucketListReply&>(obj); const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); buf.putInt(buckets.size()); @@ -417,7 +403,7 @@ RoutableFactories60::GetBucketStateMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::GetBucketStateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketStateMessage &msg = static_cast<const GetBucketStateMessage&>(obj); + const auto &msg = static_cast<const GetBucketStateMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); return true; } @@ -452,14 +438,12 @@ RoutableFactories60::GetBucketStateReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories60::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const { - return DocumentMessage::UP( - new GetDocumentMessage(decodeDocumentId(buf), - decodeString(buf))); + document::DocumentId docId = decodeDocumentId(buf); + return std::make_unique<GetDocumentMessage>(docId, decodeString(buf)); } bool -RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, - vespalib::GrowableByteBuffer &buf) const +RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); @@ -468,6 +452,17 @@ RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage & return true; } +namespace { + +std::shared_ptr<document::Document> +decodeDocument(const document::DocumentTypeRepo & repo, document::ByteBuffer & buf) { + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + auto doc = std::make_shared<document::Document>(repo, stream); + buf.incPos(buf.getRemaining() - stream.size()); + return doc; +} + +} DocumentReply::UP RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { @@ -476,7 +471,7 @@ RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf bool hasDocument = decodeBoolean(buf); document::Document * document = nullptr; if (hasDocument) { - auto doc = std::make_shared<document::Document>(_repo, buf); + auto doc = decodeDocument(_repo, buf); document = doc.get(); reply->setDocument(std::move(doc)); } @@ -509,7 +504,7 @@ DocumentMessage::UP RoutableFactories60::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<MapVisitorMessage>(); - msg->getData().deserialize(_repo, buf); + msg->getData().deserialize(buf); return msg; } @@ -517,11 +512,7 @@ bool RoutableFactories60::MapVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const MapVisitorMessage &msg = static_cast<const MapVisitorMessage&>(obj); - - int32_t len = msg.getData().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getData().serialize(dbuf); + msg.getData().serialize(buf); return true; } @@ -540,7 +531,7 @@ RoutableFactories60::MapVisitorReplyFactory::doEncode(const DocumentReply &, ves void RoutableFactories60::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocument(make_shared<document::Document>(_repo, buf)); + msg.setDocument(decodeDocument(_repo, buf)); msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf))); decodeTasCondition(msg, buf); } @@ -564,9 +555,6 @@ RoutableFactories60::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf { auto reply = make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); reply->setHighestModificationTimestamp(decodeLong(buf)); - - // Doing an explicit move here to force converting result to an rvalue. - // This is done automatically in GCC >= 5. return reply; } @@ -656,12 +644,8 @@ RoutableFactories60::SearchResultMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const SearchResultMessage &msg = static_cast<const SearchResultMessage&>(obj); - - int len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); + const auto & msg = static_cast<const SearchResultMessage&>(obj); + msg.serialize(buf); return true; } @@ -679,13 +663,10 @@ RoutableFactories60::QueryResultMessageFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::QueryResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const QueryResultMessage &msg = static_cast<const QueryResultMessage&>(obj); + const auto &msg = static_cast<const QueryResultMessage&>(obj); - int len = msg.getSearchResult().getSerializedSize() + msg.getDocumentSummary().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getSearchResult().serialize(dbuf); - msg.getDocumentSummary().serialize(dbuf); + msg.getSearchResult().serialize(buf); + msg.getDocumentSummary().serialize(buf); return true; } @@ -740,7 +721,7 @@ RoutableFactories60::StatBucketMessageFactory::doDecode(document::ByteBuffer &bu bool RoutableFactories60::StatBucketMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); + const auto &msg = static_cast<const StatBucketMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); buf.putString(msg.getDocumentSelection()); @@ -758,7 +739,7 @@ RoutableFactories60::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) bool RoutableFactories60::StatBucketReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const StatBucketReply &reply = static_cast<const StatBucketReply&>(obj); + const auto &reply = static_cast<const StatBucketReply&>(obj); buf.putString(reply.getResults()); return true; } @@ -789,7 +770,9 @@ RoutableFactories60::StatDocumentReplyFactory::doEncode(const DocumentReply &, v void RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, stream)); + buf.incPos(stream.rp()); msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); decodeTasCondition(msg, buf); @@ -798,7 +781,7 @@ RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMess bool RoutableFactories60::UpdateDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); + const auto &msg = static_cast<const UpdateDocumentMessage&>(obj); vespalib::nbostream stream; msg.getDocumentUpdate().serializeHEAD(stream); @@ -822,7 +805,7 @@ RoutableFactories60::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::UpdateDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const UpdateDocumentReply &reply = static_cast<const UpdateDocumentReply&>(obj); + const auto &reply = static_cast<const UpdateDocumentReply&>(obj); buf.putBoolean(reply.getWasFound()); buf.putLong(reply.getHighestModificationTimestamp()); return true; @@ -848,7 +831,7 @@ RoutableFactories60::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::VisitorInfoMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); + const auto &msg = static_cast<const VisitorInfoMessage&>(obj); buf.putInt(msg.getFinishedBuckets().size()); for (const auto & bucketId : msg.getFinishedBuckets()) { @@ -883,7 +866,7 @@ RoutableFactories60::WrongDistributionReplyFactory::doDecode(document::ByteBuffe bool RoutableFactories60::WrongDistributionReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const WrongDistributionReply &reply = static_cast<const WrongDistributionReply&>(obj); + const auto &reply = static_cast<const WrongDistributionReply&>(obj); buf.putString(reply.getSystemState()); return true; } diff --git a/eval/src/tests/ann/sift_benchmark.cpp b/eval/src/tests/ann/sift_benchmark.cpp index f20df926f24..dcfe1cf9c5c 100644 --- a/eval/src/tests/ann/sift_benchmark.cpp +++ b/eval/src/tests/ann/sift_benchmark.cpp @@ -285,6 +285,15 @@ TEST("require that HNSW via NNS api mostly works") { #endif +/** + * Before running the benchmark the ANN_SIFT1M data set must be downloaded and extracted: + * wget ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz + * tar -xf sift.tar.gz + * + * The benchmark program will load the data set from $HOME/sift if no directory is specified. + * + * More information about the dataset is found here: http://corpus-texmex.irisa.fr/. + */ int main(int argc, char **argv) { TEST_MASTER.init(__FILE__); std::string sift_dir = "."; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index eff8b48798e..0d1ef3478ef 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -173,13 +173,6 @@ public class Flags { "Takes effect on restart of process", NODE_TYPE, HOSTNAME); - public static final UnboundBooleanFlag USE_OLD_METRICS_CHECKS = defineFeatureFlag( - "use-old-metrics-checks", true, - "Whether to use old metrics checks", - "Takes effect on next host admin tick", - NODE_TYPE, HOSTNAME, APPLICATION_ID); - - public static final UnboundBooleanFlag ENABLE_DISK_WRITE_TEST = defineFeatureFlag( "enable-disk-write-test", false, "Regularly issue a small write to disk and fail the host if it is not successful", @@ -203,11 +196,6 @@ public class Flags { "Takes effect on next deployment", APPLICATION_ID); - public static final UnboundBooleanFlag FAIL_STARTING_NODE_ON_IP_MISMATCH = defineFeatureFlag( - "fail-starting-node-on-ip-mismatch", false, - "Whether node-admin should refuse to start container when there is an IP mismatch between the DNS and node-repository", - "Takes effect on next node creation (f.ex. node reboot or vespa version upgrade)"); - public static final UnboundBooleanFlag USE_CONFIG_SERVER_FOR_TESTER_API_CALLS = defineFeatureFlag( "use-config-server-for-tester-api-calls", false, "Whether controller should send requests to tester API through config server (if false) or tester endpoint (if true)", @@ -220,6 +208,12 @@ public class Flags { "Takes effect immediately", ZONE_ID, HOSTNAME); + public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag( + "generate-l4-routing-config", false, + "Whether routing nodes should generate L4 routing config", + "Takes effect immediately", + ZONE_ID, HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index caabdfa1479..f1486ae7117 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -288,11 +288,11 @@ public abstract class ControllerHttpClient { private static String metaToJson(Submission submission) { Slime slime = new Slime(); Cursor rootObject = slime.setObject(); - rootObject.setString("repository", submission.repository()); - rootObject.setString("branch", submission.branch()); - rootObject.setString("commit", submission.commit()); + submission.repository().ifPresent(repository -> rootObject.setString("repository", repository)); + submission.branch().ifPresent(branch -> rootObject.setString("branch", branch)); + submission.commit().ifPresent(commit -> rootObject.setString("commit", commit)); submission.sourceUrl().ifPresent(url -> rootObject.setString("sourceUrl", url)); - rootObject.setString("authorEmail", submission.authorEmail()); + submission.authorEmail().ifPresent(email -> rootObject.setString("authorEmail", email)); submission.projectId().ifPresent(projectId -> rootObject.setLong("projectId", projectId)); return toJson(slime); } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java index 6f392de86e7..22b0c619acc 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Submission.java @@ -12,17 +12,18 @@ import java.util.OptionalLong; */ public class Submission { - private final String repository; - private final String branch; - private final String commit; + private final Optional<String> repository; + private final Optional<String> branch; + private final Optional<String> commit; private final Optional<String> sourceUrl; - private final String authorEmail; + private final Optional<String> authorEmail; private final Path applicationZip; private final Path applicationTestZip; - private final OptionalLong projectId; + private final Optional<Long> projectId; - public Submission(String repository, String branch, String commit, Optional<String> sourceUrl, String authorEmail, - Path applicationZip, Path applicationTestZip, OptionalLong projectId) { + public Submission(Optional<String> repository, Optional<String> branch, Optional<String> commit, + Optional<String> sourceUrl, Optional<String> authorEmail, + Path applicationZip, Path applicationTestZip, Optional<Long> projectId) { this.repository = repository; this.branch = branch; this.commit = commit; @@ -33,13 +34,13 @@ public class Submission { this.projectId = projectId; } - public String repository() { return repository; } - public String branch() { return branch; } - public String commit() { return commit; } + public Optional<String> repository() { return repository; } + public Optional<String> branch() { return branch; } + public Optional<String> commit() { return commit; } public Optional<String> sourceUrl() { return sourceUrl; } - public String authorEmail() { return authorEmail; } + public Optional<String> authorEmail() { return authorEmail; } public Path applicationZip() { return applicationZip; } public Path applicationTestZip() { return applicationTestZip; } - public OptionalLong projectId() { return projectId; } + public Optional<Long> projectId() { return projectId; } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index 07070027f96..c8a8e65be5d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -13,31 +13,27 @@ import java.util.Objects; */ public class Node { - final String nodeId; - final String host; - final int port; - final String path; + public final String role; + public final String hostname; + private final int port; + private final String path; private final String metricsUriBase; public Node(MetricsNodesConfig.Node nodeConfig) { - this(nodeConfig.nodeId(), nodeConfig.hostname(), nodeConfig.metricsPort() , nodeConfig.metricsPath()); + this(nodeConfig.role(), nodeConfig.hostname(), nodeConfig.metricsPort() , nodeConfig.metricsPath()); } - public Node(String nodeId, String host, int port, String path) { - Objects.requireNonNull(nodeId, "Null configId is not allowed"); - Objects.requireNonNull(host, "Null host is not allowed"); + public Node(String role, String hostname, int port, String path) { + Objects.requireNonNull(role, "Null configId is not allowed"); + Objects.requireNonNull(hostname, "Null hostname is not allowed"); Objects.requireNonNull(path, "Null path is not allowed"); - this.nodeId = nodeId; - this.host = host; + this.role = role; + this.hostname = hostname; this.port = port; this.path = path; - metricsUriBase = "http://" + host + ":" + port + path; - } - - public String getName() { - return nodeId; + metricsUriBase = "http://" + hostname + ":" + port + path; } URI metricsUri(ConsumerId consumer) { @@ -50,12 +46,13 @@ public class Node { if (o == null || getClass() != o.getClass()) return false; Node node = (Node) o; return port == node.port && - nodeId.equals(node.nodeId) && - host.equals(node.host); + path.equals(node.path) && + role.equals(node.role) && + hostname.equals(node.hostname); } @Override public int hashCode() { - return Objects.hash(nodeId, host, port); + return Objects.hash(role, hostname, port, path); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java index bd17a238607..b7abb2c0349 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonModel.java @@ -19,11 +19,14 @@ import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(NON_ABSENT) -@JsonPropertyOrder({ "name", "node", "services" }) +@JsonPropertyOrder({ "hostname", "role", "node", "services" }) public class GenericJsonModel { - @JsonProperty("name") - public String name; + @JsonProperty("hostname") + public String hostname; + + @JsonProperty("role") + public String role; @JsonProperty("node") public GenericNode node; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java index a2125abb6a9..e249338c318 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/GenericJsonUtil.java @@ -12,7 +12,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -42,7 +41,7 @@ public class GenericJsonUtil { var genericJsonModels = new ArrayList<GenericJsonModel>(); metricsByNode.forEach( - (node, metrics) -> genericJsonModels.add(toGenericJsonModel(metrics, node.getName()))); + (node, metrics) -> genericJsonModels.add(toGenericJsonModel(metrics, node))); applicationModel.nodes = genericJsonModels; return applicationModel; @@ -52,11 +51,16 @@ public class GenericJsonUtil { return toGenericJsonModel(metricsPackets, null); } - public static GenericJsonModel toGenericJsonModel(List<MetricsPacket> metricsPackets, String nodeName) { + public static GenericJsonModel toGenericJsonModel(List<MetricsPacket> metricsPackets, Node node) { Map<ServiceId, List<MetricsPacket>> packetsByService = metricsPackets.stream() .collect(Collectors.groupingBy(packet -> packet.service, LinkedHashMap::new, toList())); var jsonModel = new GenericJsonModel(); + if (node != null) { + jsonModel.hostname = node.hostname; + jsonModel.role = node.role; + } + var genericServices = new ArrayList<GenericService>(); packetsByService.forEach((serviceId, packets) -> { var genericMetricsList = packets.stream() @@ -72,7 +76,6 @@ public class GenericJsonUtil { .get(); if (VESPA_NODE_SERVICE_ID.equals(serviceId)) { jsonModel.node = new GenericNode(genericService.timestamp, genericService.metrics); - jsonModel.name = nodeName; } else { genericServices.add(genericService); diff --git a/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def b/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def index 0de2f21d972..ca06148638f 100644 --- a/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def +++ b/metrics-proxy/src/main/resources/configdefinitions/metrics-nodes.def @@ -1,7 +1,10 @@ # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package=ai.vespa.metricsproxy.http.application -node[].nodeId string +# TODO: remove, unused +node[].nodeId string default="" + node[].hostname string +node[].role string node[].metricsPort int node[].metricsPath string diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index d1224e79e45..074b7877430 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -43,7 +43,8 @@ import static org.junit.Assert.fail; @SuppressWarnings("UnstableApiUsage") public class ApplicationMetricsHandlerTest { - private static final String URI_BASE = "http://localhost"; + private static final String HOST = "localhost"; + private static final String URI_BASE = "http://" + HOST; private static final String APP_METRICS_V1_URI = URI_BASE + V1_PATH; private static final String APP_METRICS_VALUES_URI = URI_BASE + VALUES_PATH; @@ -117,7 +118,8 @@ public class ApplicationMetricsHandlerTest { assertEquals(1, jsonModel.nodes.size()); GenericJsonModel nodeModel = jsonModel.nodes.get(0); - assertEquals(MOCK_METRICS_PATH, nodeModel.name); + assertEquals(HOST, nodeModel.hostname); + assertEquals(MOCK_METRICS_PATH, nodeModel.role); assertEquals(2, nodeModel.node.metrics.size()); assertEquals(16.222, nodeModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d); } @@ -171,7 +173,7 @@ public class ApplicationMetricsHandlerTest { private MetricsNodesConfig.Node.Builder nodeConfig(String path) { return new MetricsNodesConfig.Node.Builder() - .nodeId(path) + .role(path) .hostname("localhost") .metricsPath(path) .metricsPort(port); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java index 5ff6b580988..1f2852e3526 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java @@ -157,7 +157,7 @@ public class ApplicationMetricsRetrieverTest { private MetricsNodesConfig.Node.Builder nodeConfig(String path) { return new MetricsNodesConfig.Node.Builder() - .nodeId(path) + .role(path) .hostname(HOST) .metricsPath(path) .metricsPort(port); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java index c0abc3efb86..5a2d374af2e 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/GenericApplicationModelTest.java @@ -33,7 +33,8 @@ public class GenericApplicationModelTest { // Do some sanity checking assertEquals(2, model.nodes.size()); GenericJsonModel node0Model = model.nodes.get(0); - assertEquals("node0", node0Model.name); + assertEquals("node0", node0Model.hostname); + assertEquals("role0", node0Model.role); assertEquals(1, node0Model.services.size()); GenericService service = node0Model.services.get(0); assertEquals(1, service.metrics.size()); @@ -41,7 +42,8 @@ public class GenericApplicationModelTest { GenericJsonModel node1Model = model.nodes.get(1); GenericNode node1 = node1Model.node; - assertEquals("node1", node1Model.name); + assertEquals("node1", node1Model.hostname); + assertEquals("role1", node1Model.role); assertEquals(32.444, node1.metrics.get(0).values.get("cpu.util"), 0.001d); assertThatSerializedModelEqualsTestFile(model); @@ -63,13 +65,15 @@ public class GenericApplicationModelTest { .build(); - var metricsByNode = Map.of(toNode("node0"), List.of(nodePacket, servicePacket)); + var metricsByNode = Map.of(new Node("my-role", "hostname", 0, "path"), + List.of(nodePacket, servicePacket)); GenericApplicationModel model = GenericJsonUtil.toGenericApplicationModel(metricsByNode); GenericJsonModel nodeModel = model.nodes.get(0); assertNotNull(nodeModel.node); - assertEquals("node0", nodeModel.name); + assertEquals("hostname", nodeModel.hostname); + assertEquals("my-role", nodeModel.role); assertEquals(1, nodeModel.node.metrics.size()); GenericMetrics nodeMetrics = nodeModel.node.metrics.get(0); assertEquals(1.234, nodeMetrics.values.get("node-metric"), 0.001d); @@ -112,7 +116,4 @@ public class GenericApplicationModelTest { return mapper.readValue(getFileContents(TEST_FILE), GenericApplicationModel.class); } - private static Node toNode(String name) { - return new Node(name, "host", 0, "path"); - } } diff --git a/metrics-proxy/src/test/resources/generic-application.json b/metrics-proxy/src/test/resources/generic-application.json index 5ddd11962be..ff529a240db 100644 --- a/metrics-proxy/src/test/resources/generic-application.json +++ b/metrics-proxy/src/test/resources/generic-application.json @@ -1,7 +1,8 @@ { "nodes": [ { - "name": "node0", + "hostname": "node0", + "role": "role0", "node": { "timestamp": 1234, "metrics": [ @@ -36,7 +37,8 @@ ] }, { - "name": "node1", + "hostname": "node1", + "role": "role1", "node": { "timestamp": 1234, "metrics": [ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index d515f0d0353..9da480e55c7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.vespa.athenz.api.AthenzIdentity; import java.net.URI; -import java.util.Collections; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; @@ -27,7 +26,7 @@ public class ConfigServerInfo { this.configServerHostnameToUriMapper = hostname -> URI.create(scheme + "://" + hostname + ":" + port); this.configServerURIs = configServerHostNames.stream() .map(configServerHostnameToUriMapper) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(Collectors.toUnmodifiableList()); } private static URI createLoadBalancerEndpoint(String loadBalancerHost, String scheme, int port) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index a533556bde8..67dcb6744ce 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -77,7 +77,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { HostnameVerifier hostnameVerifier, HostName configServerHostname) { return new ConfigServerApiImpl( - Collections.singleton(info.getConfigServerUri(configServerHostname.value())), + List.of(info.getConfigServerUri(configServerHostname.value())), hostnameVerifier, provider); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java index 8f39fddfa1f..58c9c197d65 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java @@ -61,9 +61,9 @@ public class Acl { rules.add("-A INPUT -p " + ipVersion.icmpProtocol() + " -j ACCEPT"); // Allow trusted ports if any - String commaSeparatedPorts = trustedPorts.stream().map(i -> Integer.toString(i)).sorted().collect(Collectors.joining(",")); - if (!commaSeparatedPorts.isEmpty()) { - rules.add("-A INPUT -p tcp -m multiport --dports " + commaSeparatedPorts + " -j ACCEPT"); + if (!trustedPorts.isEmpty()) { + String ports = trustedPorts.stream().map(i -> Integer.toString(i)).sorted().collect(Collectors.joining(",")); + rules.add("-A INPUT -p tcp -m multiport --dports " + ports + " -j ACCEPT"); } // Allow traffic from trusted nodes diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java index ea2f313f03a..2d59898dc52 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetAclResponse.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Collections; import java.util.List; /** @@ -29,9 +28,9 @@ public class GetAclResponse { public GetAclResponse(@JsonProperty("trustedNodes") List<Node> trustedNodes, @JsonProperty("trustedNetworks") List<Network> trustedNetworks, @JsonProperty("trustedPorts") List<Port> trustedPorts) { - this.trustedNodes = trustedNodes == null ? Collections.emptyList() : trustedNodes; - this.trustedNetworks = trustedNetworks == null ? Collections.emptyList() : trustedNetworks; - this.trustedPorts = trustedPorts == null ? Collections.emptyList() : trustedPorts; + this.trustedNodes = trustedNodes == null ? List.of() : trustedNodes; + this.trustedNetworks = trustedNetworks == null ? List.of() : trustedNetworks; + this.trustedPorts = trustedPorts == null ? List.of() : trustedPorts; } @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index ef9e33d20d4..c790e73037e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -6,9 +6,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.dockerapi.Container; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; @@ -52,13 +50,11 @@ public class DockerOperationsImpl implements DockerOperations { private final Docker docker; private final Terminal terminal; private final IPAddresses ipAddresses; - private final BooleanFlag failStartingNodeOnIpMismatch; public DockerOperationsImpl(Docker docker, Terminal terminal, IPAddresses ipAddresses, FlagSource flagSource) { this.docker = docker; this.terminal = terminal; this.ipAddresses = ipAddresses; - this.failStartingNodeOnIpMismatch = Flags.FAIL_STARTING_NODE_ON_IP_MISMATCH.bindTo(flagSource); } @Override @@ -96,10 +92,8 @@ public class DockerOperationsImpl implements DockerOperations { Optional<? extends InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname()); Optional<? extends InetAddress> ipV6Local = ipAddresses.getIPv6Address(context.node().hostname()); - if (failStartingNodeOnIpMismatch.value()) { - assertEqualIpAddresses(context.hostname(), ipV4Local, context.node().ipAddresses(), IPVersion.IPv4); - assertEqualIpAddresses(context.hostname(), ipV6Local, context.node().ipAddresses(), IPVersion.IPv6); - } + assertEqualIpAddresses(context.hostname(), ipV4Local, context.node().ipAddresses(), IPVersion.IPv4); + assertEqualIpAddresses(context.hostname(), ipV6Local, context.node().ipAddresses(), IPVersion.IPv6); if (ipV4Local.isEmpty() && ipV6Local.isEmpty()) { throw new ConvergenceException("Container " + context.node().hostname() + " with " + networking + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java index a9954200f8a..65941ed533a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import java.util.LinkedList; import java.util.List; /** @@ -16,10 +15,11 @@ import java.util.List; */ class FilterTableLineEditor implements LineEditor { - private final LinkedList<String> wantedRules; + private final List<String> wantedRules; + private int position = 0; private FilterTableLineEditor(List<String> wantedRules) { - this.wantedRules = new LinkedList<>(wantedRules); + this.wantedRules = List.copyOf(wantedRules); } static FilterTableLineEditor from(Acl acl, IPVersion ipVersion) { @@ -29,15 +29,33 @@ class FilterTableLineEditor implements LineEditor { @Override public LineEdit edit(String line) { - // We have already added all the lines we wanted, remove the remainer - if (wantedRules.isEmpty()) return LineEdit.remove(); - - String wantedRule = wantedRules.pop(); - return wantedRule.equals(line) ? LineEdit.none() : LineEdit.replaceWith(wantedRule); + int index = indexOf(wantedRules, line, position); + // Unwanted rule, remove + if (index < 0) return LineEdit.remove(); + + // Wanted rule at the expected position, no diff + if (index == position) { + position++; + return LineEdit.none(); + } + + // Insert the rules between position and index before index + List<String> toInsert = wantedRules.subList(position, index); + position = ++index; + return LineEdit.insertBefore(toInsert); } @Override public List<String> onComplete() { - return this.wantedRules; + return wantedRules.subList(position, wantedRules.size()); + } + + private static <T> int indexOf(List<T> list, T value, int startPos) { + for (int i = startPos; i < list.size(); i++) { + if (value.equals(list.get(i))) + return i; + } + + return -1; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java index e1dc4a661ff..be08e1b2aec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditor.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -45,7 +43,7 @@ class NatTableLineEditor implements LineEditor { @Override public List<String> onComplete() { - if (redirectExists) return new ArrayList<>(); - return Collections.singletonList(redirectRule); + if (redirectExists) return List.of(); + return List.of(redirectRule); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index c063059e8a9..cdf5687d61a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -82,7 +82,7 @@ public class CoreCollector { if (result.getExitStatus() != 0) throw new RuntimeException("Failed to read backtrace " + result + ", Command: " + Arrays.toString(command)); - return Arrays.asList(result.getOutput().split("\n")); + return List.of(result.getOutput().split("\n")); } List<String> readJstack(NodeAgentContext context, Path coredumpPath, Path binPath) { @@ -92,7 +92,7 @@ public class CoreCollector { if (result.getExitStatus() != 0) throw new RuntimeException("Failed to read jstack " + result + ", Command: " + Arrays.toString(command)); - return Arrays.asList(result.getOutput().split("\n")); + return List.of(result.getOutput().split("\n")); } /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 2874546da52..9b0a35d4b96 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -159,7 +158,7 @@ public class CoredumpHandler { Map<String, Object> metadata = coreCollector.collect(context, coredumpFilePathInContainer); metadata.putAll(nodeAttributesSupplier.get()); - String metadataFields = objectMapper.writeValueAsString(ImmutableMap.of("fields", metadata)); + String metadataFields = objectMapper.writeValueAsString(Map.of("fields", metadata)); metadataPath.writeUtf8File(metadataFields); return metadataFields; } else { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 17dc61978cf..4c5ad7c5c3a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -44,11 +44,10 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; -import static java.util.Collections.singleton; - /** * A maintainer that is responsible for providing and refreshing Athenz credentials for a container. * @@ -103,7 +102,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { this.identityDocumentClient = new DefaultIdentityDocumentClient( configServerInfo.getLoadBalancerEndpoint(), hostIdentityProvider, - new AthenzIdentityVerifier(singleton(configserverIdentity))); + new AthenzIdentityVerifier(Set.of(configserverIdentity))); this.clock = clock; this.useInternalZts = useInternalZts; } @@ -194,7 +193,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis HostnameVerifier ztsHostNameVerifier = useInternalZts - ? new AthenzIdentityVerifier(singleton(configserverIdentity)) + ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) : null; try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, hostIdentityProvider, ztsHostNameVerifier)) { InstanceIdentity instanceIdentity = @@ -223,7 +222,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { try { // Set up a hostname verified for zts if this is configured to use the config server (internal zts) apis HostnameVerifier ztsHostNameVerifier = useInternalZts - ? new AthenzIdentityVerifier(singleton(configserverIdentity)) + ? new AthenzIdentityVerifier(Set.of(configserverIdentity)) : null; try (ZtsClient ztsClient = new DefaultZtsClient(ztsEndpoint, containerIdentitySslContext, ztsHostNameVerifier)) { InstanceIdentity instanceIdentity = diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index ca6a56413c3..caffe5ef2f1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -47,20 +47,19 @@ public class NodeAdminImpl implements NodeAdmin { private final Counter numberOfUnhandledExceptions; public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock) { - this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), + this(nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), metrics, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD); } public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread) { - this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), + this(nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), metrics, clock, freezeTimeout, spread); } NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread) { this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; - this.clock = clock; this.freezeTimeout = freezeTimeout; this.spread = spread; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index fdb9428dc3f..68178418e62 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -428,7 +428,7 @@ public class NodeAgentImpl implements NodeAgent { } container = removeContainerIfNeededUpdateContainerState(context, container); credentialsMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - if (! container.isPresent()) { + if (container.isEmpty()) { containerState = STARTING; startContainer(context); containerState = UNKNOWN; @@ -497,7 +497,7 @@ public class NodeAgentImpl implements NodeAgent { private Optional<Container> getContainer(NodeAgentContext context) { if (containerState == ABSENT) return Optional.empty(); Optional<Container> container = dockerOperations.getContainer(context); - if (! container.isPresent()) containerState = ABSENT; + if (container.isEmpty()) containerState = ABSENT; return container; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java index 47c710395f5..f5b5c8ae31e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/CursorImpl.java @@ -1,8 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.hosted.node.admin.task.util.editor; -import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; @@ -223,7 +222,7 @@ public class CursorImpl implements Cursor { @Override public Cursor writeLines(String... lines) { - return writeLines(Arrays.asList(lines)); + return writeLines(List.of(lines)); } @Override @@ -307,7 +306,7 @@ public class CursorImpl implements Cursor { @Override public boolean replaceMatch(Pattern pattern, Function<Match, String> replacer) { Optional<Match> match = moveForwardToStartOfMatch(pattern); - if (!match.isPresent()) { + if (match.isEmpty()) { return false; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java index 16eb1a9b509..a781615e44d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/AttributeSync.java @@ -101,7 +101,7 @@ public class AttributeSync { Optional<String> wantedValue, Supplier<String> currentValueSupplier, Consumer<String> valueSetter) { - if (!wantedValue.isPresent()) { + if (wantedValue.isEmpty()) { return false; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java index 112197101b4..2b66c5fad2e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Editor.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; -import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index b78d155bdbf..f10ebd5e315 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -14,9 +14,9 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.Stack; import java.util.function.Consumer; import java.util.function.Predicate; @@ -139,7 +139,7 @@ public class FileFinder { try { // Only need to traverse as deep as we want to match, unless we want to match everything in directories // already matched - Files.walkFileTree(basePath, Collections.emptySet(), maxDepth, new SimpleFileVisitor<>() { + Files.walkFileTree(basePath, Set.of(), maxDepth, new SimpleFileVisitor<>() { private final Stack<FileAttributes> matchingDirectoryStack = new Stack<>(); private int currentLevel = -1; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java index 78e7a3e71b6..88bf25c65a9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java @@ -1,11 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; -import com.google.common.collect.ImmutableList; - import javax.annotation.concurrent.Immutable; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.REPLACE; @@ -18,17 +14,17 @@ import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.NON public class LineEdit { enum Type { NONE, REPLACE } - public static LineEdit none() { return insert(Collections.emptyList(), Collections.emptyList()); } - public static LineEdit remove() { return replaceWith(Collections.emptyList()); } + public static LineEdit none() { return insert(List.of(), List.of()); } + public static LineEdit remove() { return replaceWith(List.of()); } - public static LineEdit insertBefore(String... prepend) { return insertBefore(Arrays.asList(prepend)); } - public static LineEdit insertBefore(List<String> prepend) { return insert(prepend, Collections.emptyList()); } - public static LineEdit insertAfter(String... append) { return insertAfter(Arrays.asList(append)); } - public static LineEdit insertAfter(List<String> append) { return insert(Collections.emptyList(), append); } + public static LineEdit insertBefore(String... prepend) { return insertBefore(List.of(prepend)); } + public static LineEdit insertBefore(List<String> prepend) { return insert(prepend, List.of()); } + public static LineEdit insertAfter(String... append) { return insertAfter(List.of(append)); } + public static LineEdit insertAfter(List<String> append) { return insert(List.of(), append); } public static LineEdit insert(List<String> prepend, List<String> append) { return new LineEdit(NONE, prepend, append); } - public static LineEdit replaceWith(String... lines) { return replaceWith(Arrays.asList(lines)); } - public static LineEdit replaceWith(List<String> insertLines) { return new LineEdit(REPLACE, Collections.emptyList(), insertLines); } + public static LineEdit replaceWith(String... lines) { return replaceWith(List.of(lines)); } + public static LineEdit replaceWith(List<String> insertLines) { return new LineEdit(REPLACE, List.of(), insertLines); } private final Type type; private final List<String> prependLines; @@ -36,8 +32,8 @@ public class LineEdit { private LineEdit(Type type, List<String> prependLines, List<String> appendLines) { this.type = type; - this.prependLines = ImmutableList.copyOf(prependLines); - this.appendLines = ImmutableList.copyOf(appendLines); + this.prependLines = List.copyOf(prependLines); + this.appendLines = List.copyOf(appendLines); } public Type getType() { return type; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index 668c2787513..268e0a5ccfd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -19,7 +19,6 @@ import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import java.time.Instant; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -217,7 +216,7 @@ public class UnixPath { .map(UnixPath::new) .collect(Collectors.toList()); } catch (NoSuchFileException ignored) { - return Collections.emptyList(); + return List.of(); } catch (IOException e) { throw new RuntimeException("Failed to list contents of directory " + path.toAbsolutePath(), e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java index 2854dc55af8..0c03dc7f483 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java @@ -89,7 +89,7 @@ public class ChildProcess2Impl implements ChildProcess2 { @Override public void close() { try { - if ( ! commandLine.getOutputFile().isPresent()) + if (commandLine.getOutputFile().isEmpty()) Files.delete(outputPath); } catch (Throwable t) { logger.log(LogLevel.WARNING, "Failed to delete " + outputPath, t); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java index 8f39c2d257b..d86c8745ceb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java @@ -9,7 +9,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -67,7 +66,7 @@ public class CommandLine { } /** Add arguments to the command. The first argument in the first call to add() is the program. */ - public CommandLine add(String... arguments) { return add(Arrays.asList(arguments)); } + public CommandLine add(String... arguments) { return add(List.of(arguments)); } /** Add arguments to the command. The first argument in the first call to add() is the program. */ public CommandLine add(Collection<String> arguments) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java index a5f8e667ff2..dbe6b984211 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java @@ -85,7 +85,7 @@ public class ProcessFactoryImpl implements ProcessFactory { return new ChildProcess2Impl(commandLine, process, outputFile, timer); } catch (RuntimeException | Error throwable) { try { - if ( ! commandLine.getOutputFile().isPresent()) + if (commandLine.getOutputFile().isEmpty()) Files.delete(outputFile); } catch (IOException ioException) { logger.log(LogLevel.WARNING, "Failed to delete temporary file at " + diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java index 56a2b2aeca2..c7264c2fe4d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java @@ -15,7 +15,7 @@ public class SystemCtlTester extends SystemCtl { private final Set<String> runningUnits = new HashSet<>(); - private TestTerminal terminal; + private final TestTerminal terminal; public SystemCtlTester(TestTerminal terminal) { super(terminal); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java index fb85815c70f..54c8719bceb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java @@ -244,10 +244,10 @@ public class YumPackageName { public boolean isSubsetOf(YumPackageName other) { return Objects.equals(name, other.name) && - (!epoch.isPresent() || Objects.equals(epoch, other.epoch)) && - (!version.isPresent() || Objects.equals(version, other.version)) && - (!release.isPresent() || Objects.equals(release, other.release)) && - (!architecture.isPresent() || Objects.equals(architecture, other.architecture)); + (epoch.isEmpty() || Objects.equals(epoch, other.epoch)) && + (version.isEmpty() || Objects.equals(version, other.version)) && + (release.isEmpty() || Objects.equals(release, other.release)) && + (architecture.isEmpty() || Objects.equals(architecture, other.architecture)); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index 0909a03749e..a11fdc903e7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -19,9 +19,7 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.Arrays; import java.util.List; -import java.util.Optional; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; @@ -54,7 +52,7 @@ public class ConfigServerApiImplTest { private final String uri1 = "http://host1:666"; private final String uri2 = "http://host2:666"; - private final List<URI> configServers = Arrays.asList(URI.create(uri1), URI.create(uri2)); + private final List<URI> configServers = List.of(URI.create(uri1), URI.create(uri2)); private final StringBuilder mockLog = new StringBuilder(); private ConfigServerApiImpl configServerApi; @@ -122,7 +120,7 @@ public class ConfigServerApiImplTest { params.setConnectionTimeout(Duration.ofSeconds(3)); try { - TestPojo testPojo = configServerApi.get("/path", TestPojo.class, params); + configServerApi.get("/path", TestPojo.class, params); fail(); } catch (ConnectionException e) { assertNotNull(e.getCause()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java index 3010586e708..341f2b8ca02 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; import org.junit.Test; import java.util.Arrays; -import java.util.Collections; import java.util.Set; import java.util.stream.Collectors; @@ -20,12 +19,12 @@ public class AclTest { private static final Acl aclCommon = new Acl( Set.of(1234, 453), testNodes("192.1.2.2", "fb00::1", "fe80::2", "fe80::3"), - Collections.emptySet()); + Set.of()); private static final Acl aclWithoutPorts = new Acl( - Collections.emptySet(), + Set.of(), testNodes("192.1.2.2", "fb00::1", "fe80::2"), - Collections.emptySet()); + Set.of()); @Test public void no_trusted_ports() { @@ -80,7 +79,7 @@ public class AclTest { Acl aclCommonDifferentOrder = new Acl( Set.of(453, 1234), testNodes("fe80::2", "192.1.2.2", "fb00::1", "fe80::3"), - Collections.emptySet()); + Set.of()); for (IPVersion ipVersion: IPVersion.values()) { assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 9b392395045..4e0fd95384c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.net.ServerSocket; import java.net.URI; import java.time.Instant; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -67,7 +66,7 @@ public class RealNodeRepositoryTest { int port = findRandomOpenPort(); container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port), Networking.enable); ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( - Collections.singletonList(URI.create("http://127.0.0.1:" + port))); + List.of(URI.create("http://127.0.0.1:" + port))); waitForJdiscContainerToServe(configServerApi); return; } catch (RuntimeException e) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java index 936a7bb224d..d118da07247 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImplTest.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; import org.junit.Test; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -109,7 +108,7 @@ public class OrchestratorImplTest { @Test public void testBatchSuspendCall() { String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = Arrays.asList("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); + List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); when(configServerApi.put( eq("/orchestrator/v1/suspensions/hosts/host1.test.yahoo.com?hostname=a1.host1.test.yahoo.com&hostname=a2.host1.test.yahoo.com"), @@ -124,7 +123,7 @@ public class OrchestratorImplTest { @Test(expected=OrchestratorException.class) public void testBatchSuspendCallWithFailureReason() { String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = Arrays.asList("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); + List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); String failureReason = "Failed to suspend"; when(configServerApi.put( @@ -140,7 +139,7 @@ public class OrchestratorImplTest { @Test(expected=RuntimeException.class) public void testBatchSuspendCallWithSomeException() { String parentHostName = "host1.test.yahoo.com"; - List<String> hostNames = Arrays.asList("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); + List<String> hostNames = List.of("a1.host1.test.yahoo.com", "a2.host1.test.yahoo.com"); String exceptionMessage = "Exception: Something crashed!"; when(configServerApi.put( diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index 625166a10d2..80069b38748 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeReposit import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,7 +45,7 @@ public class NodeRepoMock implements NodeRepository { @Override public Map<String, Acl> getAcls(String hostname) { - return Collections.emptyMap(); + return Map.of(); } @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index 4a232a5b2bd..d1778982043 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; import org.junit.Test; -import java.util.Arrays; +import java.util.List; import java.util.OptionalLong; import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.HOST_HOSTNAME; @@ -40,7 +40,7 @@ public class RebootTest { } catch (RuntimeException ignored) { } tester.inOrder(tester.orchestrator).suspend( - eq(HOST_HOSTNAME.value()), eq(Arrays.asList(hostname, HOST_HOSTNAME.value()))); + eq(HOST_HOSTNAME.value()), eq(List.of(hostname, HOST_HOSTNAME.value()))); tester.inOrder(tester.docker).executeInContainerAsUser( eq(new ContainerName("host1")), eq("root"), eq(OptionalLong.empty()), eq(NODE_PROGRAM), eq("stop")); assertTrue(tester.nodeAdmin.setFrozen(true)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index 90674f86907..adecc370b85 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -15,8 +15,6 @@ import org.junit.Test; import java.nio.file.FileSystem; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.function.Function; @@ -70,7 +68,7 @@ public class AclMaintainerTest { verify(dockerOperations, times(2)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); verifyNoMoreInteractions(dockerOperations); - List<String> expected = Arrays.asList( + List<String> expected = List.of( // IPv4 filter table restore "*filter\n" + "-P INPUT ACCEPT\n" + @@ -137,7 +135,7 @@ public class AclMaintainerTest { verify(dockerOperations, times(1)).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); verifyNoMoreInteractions(dockerOperations); - List<String> expected = Arrays.asList( + List<String> expected = List.of( // IPv4 filter table restore "*filter\n" + "-P INPUT ACCEPT\n" + @@ -194,7 +192,7 @@ public class AclMaintainerTest { verify(dockerOperations, never()).executeCommandInNetworkNamespace(eq(context), eq("ip6tables-restore"), any()); //we don't have a ip4 address for the container so no redirect verifyNoMoreInteractions(dockerOperations); - List<String> expected = Collections.singletonList( + List<String> expected = List.of( "*filter\n" + "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java index f72cde92839..495c1318b71 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditorTest.java @@ -6,7 +6,8 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; import org.junit.Test; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; import static org.junit.Assert.assertEquals; @@ -37,14 +38,48 @@ public class FilterTableLineEditorTest { "-A INPUT -j REJECT --reject-with icmp6-port-unreachable"); } + @Test + public void produces_minimal_diff_simple() { + assertFilterTableDiff(List.of(2, 5, 3, 6, 1, 4), List.of(2, 5, 6, 1, 4), + "Patching file table:\n" + + "--A INPUT -s 2001::3/128 -j ACCEPT\n"); + } + + @Test + public void produces_minimal_diff_complex() { + assertFilterTableDiff(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), List.of(5, 11, 6, 3, 10, 4, 8, 12), + "Patching file table:\n" + + "--A INPUT -s 2001::1/128 -j ACCEPT\n" + + "--A INPUT -s 2001::2/128 -j ACCEPT\n" + + "+-A INPUT -s 2001::11/128 -j ACCEPT\n" + + "+-A INPUT -s 2001::12/128 -j ACCEPT\n" + + "--A INPUT -s 2001::7/128 -j ACCEPT\n" + + "--A INPUT -s 2001::9/128 -j ACCEPT\n"); + } + private static void assertFilterTableLineEditorResult( Acl acl, IPVersion ipVersion, String currentFilterTable, String expectedRestoreFileContent) { FilterTableLineEditor filterLineEditor = FilterTableLineEditor.from(acl, ipVersion); Editor editor = new Editor( "nat-table", - () -> Arrays.asList(currentFilterTable.split("\n")), + () -> List.of(currentFilterTable.split("\n")), result -> assertEquals(expectedRestoreFileContent, String.join("\n", result)), filterLineEditor); editor.edit(m -> {}); } + + private static void assertFilterTableDiff(List<Integer> currentIpSuffix, List<Integer> wantedIpSuffix, String diff) { + Acl.Builder currentAcl = new Acl.Builder(); + currentIpSuffix.forEach(i -> currentAcl.withTrustedNode("host" + i, "2001::" + i)); + List<String> currentTable = new ArrayList<>(); + + Acl.Builder wantedAcl = new Acl.Builder(); + wantedIpSuffix.forEach(i -> wantedAcl.withTrustedNode("host" + i, "2001::" + i)); + + new Editor("table", List::of, currentTable::addAll, FilterTableLineEditor.from(currentAcl.build(), IPVersion.IPv6)) + .edit(log -> {}); + + new Editor("table", () -> currentTable, result -> {}, FilterTableLineEditor.from(wantedAcl.build(), IPVersion.IPv6)) + .edit(log -> assertEquals(diff, log)); + } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java index 63dc69a180c..1ea3658db76 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/NatTableLineEditorTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; import org.junit.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; @@ -88,7 +88,7 @@ public class NatTableLineEditorTest { NatTableLineEditor natLineEditor = NatTableLineEditor.from(redirectRule); Editor editor = new Editor( "nat-table", - () -> Arrays.asList(currentNatTable.split("\n")), + () -> List.of(currentNatTable.split("\n")), result -> assertEquals(expectedNatTable, String.join("\n", result)), natLineEditor); editor.edit(m -> {}); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index 5aeccb4ab7d..3771774c9a5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -117,7 +117,7 @@ public class NodeAgentContextManagerTest { return new NodeAgentContextImpl.Builder("container-123.domain.tld").build(); } - private class AsyncExecutor<T> { + private static class AsyncExecutor<T> { private final Object monitor = new Object(); private final Thread thread; private volatile Optional<T> response = Optional.empty(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java index a930e2babff..afb0c86ba2a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelperTest.java @@ -1,10 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.hosted.node.admin.provider; import org.junit.Test; -import java.util.Collections; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -14,7 +13,7 @@ public class DebugHandlerHelperTest { DebugHandlerHelper helper = new DebugHandlerHelper(); helper.addConstant("constant-key", "constant-value"); - NodeAdminDebugHandler handler = () -> Collections.singletonMap("handler-value-key", "handler-value-value"); + NodeAdminDebugHandler handler = () -> Map.of("handler-value-key", "handler-value-value"); helper.addHandler("handler-key", handler); helper.addThreadSafeSupplier("supplier-key", () -> "supplier-value"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java index 3d492c41d0f..06cb382a5c4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/EditorTest.java @@ -1,5 +1,4 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; @@ -8,7 +7,6 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import java.nio.file.FileSystem; -import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; @@ -16,7 +14,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockingDetails; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -62,8 +59,8 @@ public class EditorTest { LineEdit.insertBefore("first"), // insert first, and keep the second line LineEdit.replaceWith("third", "fourth"), // remove eight, and replace with third and fourth instead LineEdit.none(), // Keep fifth - LineEdit.insert(Collections.singletonList("sixth"), // insert sixth before seventh - Collections.singletonList("eight"))); // add eight after seventh + LineEdit.insert(List.of("sixth"), // insert sixth before seventh + List.of("eight"))); // add eight after seventh Editor editor = new Editor(path.toPath(), lineEditor); TaskContext context = mock(TaskContext.class); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java index 1308e4ead8c..197077159d1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -19,7 +19,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -157,7 +156,7 @@ public class FileFinderTest { }); return paths; } catch (NoSuchFileException e) { - return Collections.emptyList(); + return List.of(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java index 8c73d522f1d..1439517bb53 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; import java.nio.file.FileSystem; -import java.nio.file.Path; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java index a141faf290b..cbf471bd611 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java @@ -11,7 +11,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertArrayEquals; @@ -71,7 +70,7 @@ public class FileSyncTest { fileData.getPermissions().ifPresent(permissions -> assertEquals(permissions, unixPath.getPermissions())); List<String> actualMods = taskContext.getSystemModificationLog(); - List<String> expectedMods = Arrays.asList(systemModificationMessages); + List<String> expectedMods = List.of(systemModificationMessages); assertEquals(expectedMods, actualMods); UnixPath unixPath = new UnixPath(path); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java index b714ab539f6..c28aa6faa30 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java @@ -1,5 +1,4 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; @@ -10,8 +9,7 @@ import java.io.UncheckedIOException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; -import java.util.Arrays; -import java.util.Collections; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -56,11 +54,11 @@ public class MakeDirectoryTest { .withGroup(group); assertTrue(makeDirectory.converge(context)); - assertEquals(Arrays.asList(modifications), context.getSystemModificationLog()); + assertEquals(List.of(modifications), context.getSystemModificationLog()); context.clearSystemModificationLog(); assertFalse(makeDirectory.converge(context)); - assertEquals(Collections.emptyList(), context.getSystemModificationLog()); + assertEquals(List.of(), context.getSystemModificationLog()); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java index 7bdf9a44ec7..6dd5087e8a0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java @@ -6,7 +6,6 @@ import org.junit.After; import org.junit.Test; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Predicate; @@ -43,7 +42,7 @@ public class CommandLineTest { assertEquals(0, result.getExitCode()); assertEquals("line1\nline2", result.getOutput()); assertEquals("line1\nline2\n\n", result.getUntrimmedOutput()); - assertEquals(Arrays.asList("line1", "line2"), result.getOutputLines()); + assertEquals(List.of("line1", "line2"), result.getOutputLines()); assertEquals(1, context.getSystemModificationLog().size()); assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index eab8bb68863..9cc4b3f3bdb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -65,9 +65,10 @@ public class GroupPreparer { boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) .value(); + // Do not in-place resize in dynamically provisioned zones boolean inPlaceResizeEnabled = enableInPlaceResize .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value(); + .value() && !dynamicProvisioningEnabled; try (Mutex lock = nodeRepository.lock(application)) { diff --git a/parent/pom.xml b/parent/pom.xml index ca6e361899c..b1ca2539ef5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -739,7 +739,7 @@ <apache.httpcore.version>4.4.12</apache.httpcore.version> <asm.version>7.0</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.version>1.8.29</athenz.version> + <athenz.version>1.8.44</athenz.version> <aws.sdk.version>1.11.542</aws.sdk.version> <!-- WARNING: If you change curator version, you also need to update zkfacade/src/main/java/org/apache/curator/**/package-info.java diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 937431fb7dd..5c8b5b029d2 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -260,7 +260,7 @@ verifyDocs(const std::vector<DocAndTimestamp>& wanted, << entry.getDocument()->toString(true); } EXPECT_EQ(wanted[wantedIdx].timestamp, entry.getTimestamp()); - size_t serSize = wanted[wantedIdx].doc->serialize()->getLength(); + size_t serSize = wanted[wantedIdx].doc->serialize().size(); EXPECT_EQ(serSize + sizeof(DocEntry), size_t(entry.getSize())); EXPECT_EQ(serSize, size_t(entry.getDocumentSize())); ++wantedIdx; diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp index d482d144d73..f46be6f3a25 100644 --- a/persistence/src/vespa/persistence/spi/docentry.cpp +++ b/persistence/src/vespa/persistence/spi/docentry.cpp @@ -2,6 +2,7 @@ #include "docentry.h" #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/objects/nbostream.h> #include <sstream> #include <cassert> @@ -10,16 +11,13 @@ namespace storage::spi { DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc) : _timestamp(t), _metaFlags(metaFlags), - _persistedDocumentSize(doc->getSerializedSize()), + _persistedDocumentSize(doc->serialize().size()), _size(_persistedDocumentSize + sizeof(DocEntry)), _documentId(), _document(std::move(doc)) { } -DocEntry::DocEntry(Timestamp t, - int metaFlags, - DocumentUP doc, - size_t serializedDocumentSize) +DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc, size_t serializedDocumentSize) : _timestamp(t), _metaFlags(metaFlags), _persistedDocumentSize(serializedDocumentSize), diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index ca8274aab00..54fc072c2b1 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -298,7 +298,7 @@ public: { } virtual RPC::Result receive(const Packet &packet) override { - vespalib::nbostream_longlivedbuf handle(packet.getHandle().c_str(), packet.getHandle().size()); + vespalib::nbostream_longlivedbuf handle(packet.getHandle().data(), packet.getHandle().size()); try { while (handle.size() > 0) { Packet::Entry entry; diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp index e5ac6ed15b0..6d7bd243e71 100644 --- a/searchcore/src/tests/grouping/grouping.cpp +++ b/searchcore/src/tests/grouping/grouping.cpp @@ -168,7 +168,7 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock, f1.timeOfDoom, os.c_str(), os.size()); + GroupingContext context(f1.clock, f1.timeOfDoom, os.data(), os.size()); ASSERT_TRUE(!context.empty()); GroupingContext::GroupingList list = context.getGroupingList(); ASSERT_TRUE(list.size() == 1); @@ -226,7 +226,7 @@ TEST_F("testGroupingContextSerializing", DoomFixture()) { context.serialize(); vespalib::nbostream & res(context.getResult()); EXPECT_EQUAL(res.size(), os.size()); - ASSERT_TRUE(memcmp(res.c_str(), os.c_str(), res.size()) == 0); + ASSERT_TRUE(memcmp(res.data(), os.data(), res.size()) == 0); } TEST_F("testGroupingManager", DoomFixture()) { diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 8c4bf0d55b0..95ab43dbcba 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -616,7 +616,7 @@ TEST("require that grouping is performed (multi-threaded)") { Grouping grequest; grequest.setRoot(Group().addResult(SumAggregationResult().setExpression(createAttr()))); grequest.serialize(os); - request->groupSpec.assign(buf.c_str(), buf.c_str() + buf.size()); + request->groupSpec.assign(buf.data(), buf.data() + buf.size()); } SearchReply::UP reply = world.performSearch(request, threads); { diff --git a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp index 109a4cc7a25..c3338a973c4 100644 --- a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp +++ b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp @@ -36,7 +36,7 @@ private: void insert_tensor_in_properties(const vespalib::string& tensor_name, const Value& tensor_value) { vespalib::nbostream stream; DefaultTensorEngine::ref().encode(tensor_value, stream); - _props.add(tensor_name, vespalib::stringref(stream.c_str(), stream.size())); + _props.add(tensor_name, vespalib::stringref(stream.data(), stream.size())); } public: diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 96096c0401f..ca48fb773d8 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -105,7 +105,7 @@ RemoveOperationContext::RemoveOperationContext(search::SerialNum serial) str(), packet() { op.serialize(str); - ConstBufferRef buf(str.c_str(), str.wp()); + ConstBufferRef buf(str.data(), str.wp()); packet.reset(new Packet()); packet->add(Packet::Entry(serial, FeedOperation::REMOVE, buf)); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp index 24ae86760c9..d2490985e77 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp @@ -94,7 +94,7 @@ LidStateVector::setBit(unsigned int idx) _highest = idx; } assert(!_bv.testBit(idx)); - _bv.slowSetBit(idx); + _bv.setBitAndMaintainCount(idx); ++_count; assert(_count == internalCount()); } @@ -105,7 +105,7 @@ LidStateVector::clearBit(unsigned int idx) { assert(idx < _bv.size()); assert(_bv.testBit(idx)); - _bv.slowClearBit(idx); + _bv.clearBitAndMaintainCount(idx); --_count; assert(_count == internalCount()); maybeUpdateLowest(); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index e658bc0cfa0..1bcbe4e9683 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -31,7 +31,7 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss if (removed) { return new DocEntry(timestamp, storage::spi::REMOVE_ENTRY, doc->getId()); } else { - ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->getSerializedSize(); + ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->serialize().size(); return new DocEntry(timestamp, storage::spi::NONE, std::move(doc), serializedSize); } } else { diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index 5a6a990df9b..faee5914a97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -49,7 +49,7 @@ handleProgress(TlsReplayProgress &progress, SerialNum currentSerial) void handlePacket(PacketWrapper::SP wrap, EntryHandler entryHandler) { - vespalib::nbostream_longlivedbuf handle(wrap->packet.getHandle().c_str(), wrap->packet.getHandle().size()); + vespalib::nbostream_longlivedbuf handle(wrap->packet.getHandle().data(), wrap->packet.getHandle().size()); while (handle.size() > 0) { Packet::Entry entry; entry.deserialize(handle); diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp index bfc59dee35e..bbd02d7efce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp @@ -14,7 +14,7 @@ namespace proton { void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf, DoneCallback onDone) { - Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.c_str(), buf.size())); + Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.data(), buf.size())); Packet packet; packet.add(entry); packet.close(); diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index 5990b3ec805..46aca14325f 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -78,7 +78,7 @@ generate(uint64_t serialNum, size_t chunks, FastOS_FileInterface & idxFile, size fprintf(stdout, "Failed with lengthError %ld due to '%s'\n", lengthError, e.what()); } } - idxFile.Write2(os.c_str(), os.size()); + idxFile.Write2(os.data(), os.size()); return serialNum; } diff --git a/searchlib/src/protobuf/search_protocol.proto b/searchlib/src/protobuf/search_protocol.proto index 6a4d3cfb07a..ded19fe132f 100644 --- a/searchlib/src/protobuf/search_protocol.proto +++ b/searchlib/src/protobuf/search_protocol.proto @@ -88,4 +88,5 @@ message MonitorReply { bool online = 1; int64 active_docs = 2; int32 distribution_key = 3; + bool is_blocking_writes = 4; } diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 513e94321e1..610fc58e98f 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -489,9 +489,9 @@ TEST("testResultNodes") { double d(786324.78); nbostream os; os << j << d; - RawResultNode r1(os.c_str(), sizeof(j)); + RawResultNode r1(os.data(), sizeof(j)); EXPECT_EQUAL(r1.getInteger(), 789); - RawResultNode r2(os.c_str() + sizeof(j), sizeof(d)); + RawResultNode r2(os.data() + sizeof(j), sizeof(d)); EXPECT_EQUAL(r2.getFloat(), 786324.78); StringResultNode s1, s2("a"), s3("a"), s4("b"), s5("bb"); @@ -560,7 +560,7 @@ void testStreaming(const Identifiable &v) { EXPECT_EQUAL(os2.size(), os3.size()); ASSERT_TRUE(os2.size() == os3.size()); - EXPECT_EQUAL(0, memcmp(os2.c_str(), os3.c_str(), os3.size())); + EXPECT_EQUAL(0, memcmp(os2.data(), os3.data(), os3.size())); } TEST("testTimeStamp") { diff --git a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp index ed681d9021b..a9df188d417 100644 --- a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp +++ b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp @@ -51,10 +51,10 @@ void BitVectorBenchmark::init(size_t n) BitVector *b(BitVector::create(n).release()); srand(1); for(size_t i(0), j(0); i < n; i += rand()%10, j++) { - a->flip(i); + a->flipBit(i); } for(size_t i(0), j(0); i < n; i += rand()%10, j++) { - b->flip(i); + b->flipBit(i); } a->invalidateCachedCount(); b->invalidateCachedCount(); diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 4cbe96c74b5..d720b105671 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -268,21 +268,21 @@ TEST("requireThatSequentialOperationsOnPartialWorks") p1.invalidateCachedCount(); EXPECT_TRUE(p1.hasTrueBits()); EXPECT_EQUAL(1u, p1.countTrueBits()); - p1.slowSetBit(718); - p1.slowSetBit(739); - p1.slowSetBit(871); - p1.slowSetBit(903); + p1.setBitAndMaintainCount(718); + p1.setBitAndMaintainCount(739); + p1.setBitAndMaintainCount(871); + p1.setBitAndMaintainCount(903); EXPECT_EQUAL(5u, p1.countTrueBits()); EXPECT_TRUE(assertBV("[718,719,739,871,903]", p1)); PartialBitVector p2(717,919); EXPECT_FALSE(p1 == p2); - p2.slowSetBit(719); - p2.slowSetBit(718); - p2.slowSetBit(739); - p2.slowSetBit(871); + p2.setBitAndMaintainCount(719); + p2.setBitAndMaintainCount(718); + p2.setBitAndMaintainCount(739); + p2.setBitAndMaintainCount(871); EXPECT_FALSE(p1 == p2); - p2.slowSetBit(903); + p2.setBitAndMaintainCount(903); EXPECT_TRUE(p1 == p2); AllocatedBitVector full(1000); @@ -422,10 +422,10 @@ TEST("requireThatSetWorks") EXPECT_EQUAL(4u, v1.countTrueBits()); EXPECT_TRUE(assertBV("[7,39,80,103]", v1)); - v1.slowSetBit(39); + v1.setBitAndMaintainCount(39); EXPECT_EQUAL(4u, v1.countTrueBits()); EXPECT_TRUE(assertBV("[7,39,80,103]", v1)); - v1.slowSetBit(57); + v1.setBitAndMaintainCount(57); EXPECT_EQUAL(5u, v1.countTrueBits()); EXPECT_TRUE(assertBV("[7,39,57,80,103]", v1)); } diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 0a3c3788c98..cc31fcec4d4 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -127,7 +127,7 @@ MyVisitor::visit(uint32_t lid, const std::shared_ptr<Document> &doc) assert(lid < _docIdLimit); Document::UP expDoc(makeDoc(_repo, lid, _before)); EXPECT_TRUE(*expDoc == *doc); - _valid->slowSetBit(lid); + _valid->setBitAndMaintainCount(lid); } @@ -136,7 +136,7 @@ MyVisitor::visit(uint32_t lid) { ++_visitRmCount; assert(lid < _docIdLimit); - _valid->slowClearBit(lid); + _valid->clearBitAndMaintainCount(lid); } @@ -158,7 +158,7 @@ MyRewriteVisitor::visit(uint32_t lid, const std::shared_ptr<Document> &doc) assert(lid < _docIdLimit); Document::UP expDoc(makeDoc(_repo, lid, _before)); EXPECT_TRUE(*expDoc == *doc); - _valid->slowSetBit(lid); + _valid->setBitAndMaintainCount(lid); doc->set("extra", "foo"); } @@ -297,7 +297,7 @@ Fixture::put(const Document &doc, uint32_t lid) ++_syncToken; assert(lid < _docIdLimit); _store->write(_syncToken, lid, doc); - _valid->slowSetBit(lid); + _valid->setBitAndMaintainCount(lid); } @@ -307,7 +307,7 @@ Fixture::remove(uint32_t lid) ++_syncToken; assert(lid < _docIdLimit); _store->remove(_syncToken, lid); - _valid->slowClearBit(lid); + _valid->clearBitAndMaintainCount(lid); } diff --git a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp index b7fb3d2b6a1..2f710c5d6e1 100644 --- a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp +++ b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp @@ -111,7 +111,7 @@ struct ArrayFixture : FixtureBase { void check_prepare_state_output(const vespalib::tensor::Tensor & tensor, const ExpectedType & expected) { vespalib::nbostream os; vespalib::tensor::TypedBinaryFormat::serialize(os, tensor); - vespalib::string input_vector(os.c_str(), os.size()); + vespalib::string input_vector(os.data(), os.size()); check_prepare_state_output(".tensor", input_vector, expected); } diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index c4751af5adb..0dced597917 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -63,27 +63,27 @@ class CallBackTest : public TransLogClient::Visitor::Callback private: virtual RPC::Result receive(const Packet & packet) override; virtual void eof() override { _eof = true; } - typedef std::map<SerialNum, ByteBuffer> PacketMap; + typedef std::map<SerialNum, std::unique_ptr<ByteBuffer>> PacketMap; PacketMap _packetMap; public: CallBackTest() : _eof(false) { } size_t size() const { return _packetMap.size(); } bool hasSerial(SerialNum n) const { return (_packetMap.find(n) != _packetMap.end()); } void clear() { _eof = false; _packetMap.clear(); } - const ByteBuffer & packet(SerialNum n) { return (_packetMap.find(n)->second); } + const ByteBuffer & packet(SerialNum n) { return *(_packetMap.find(n)->second); } bool _eof; }; RPC::Result CallBackTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(h.peek(), h.size()).c_str()); while(h.size() > 0) { Packet::Entry e; e.deserialize(h); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(e.data().c_str(), e.data().size()).c_str()); - _packetMap[e.serial()] = ByteBuffer(e.data().c_str(), e.data().size()); + _packetMap[e.serial()] = std::make_unique<ByteBuffer>(e.data().c_str(), e.data().size()); } return RPC::OK; } @@ -103,7 +103,7 @@ public: RPC::Result CallBackManyTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); for(;h.size() > 0; _count++, _value++) { Packet::Entry e; e.deserialize(h); @@ -135,7 +135,7 @@ public: RPC::Result CallBackUpdate::receive(const Packet & packet) { - nbostream_longlivedbuf h(packet.getHandle().c_str(), packet.getHandle().size()); + nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); while (h.size() > 0) { Packet::Entry e; e.deserialize(h); @@ -187,7 +187,7 @@ public: RPC::Result CallBackStatsTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); for(;h.size() > 0; ++_count) { Packet::Entry e; e.deserialize(h); @@ -236,13 +236,13 @@ bool Test::partialUpdateTest() nbostream os; os << du; - vespalib::ConstBufferRef bb(os.c_str(), os.size()); + vespalib::ConstBufferRef bb(os.data(), os.size()); LOG(info, "DU : %s", myhex(bb.c_str(), bb.size()).c_str()); Packet::Entry e(7, du.getClass().id(), bb); Packet pa; pa.add(e); pa.close(); - ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().c_str(), pa.getHandle().size()))); + ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().data(), pa.getHandle().size()))); CallBackUpdate ca; TransLogClient::Visitor::UP visitor = tls.createVisitor("test1", ca); @@ -320,10 +320,10 @@ bool Test::fillDomainTest(TransLogClient::Session * s1, const vespalib::string & ASSERT_TRUE (!b.add(e1)); a.close(); b.close(); - ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(a.getHandle().c_str(), a.getHandle().size()))); - ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(b.getHandle().c_str(), b.getHandle().size()))); + ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(a.getHandle().data(), a.getHandle().size()))); + ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(b.getHandle().data(), b.getHandle().size()))); try { - s1->commit(vespalib::ConstBufferRef(a.getHandle().c_str(), a.getHandle().size())); + s1->commit(vespalib::ConstBufferRef(a.getHandle().data(), a.getHandle().size())); ASSERT_TRUE(false); } catch (const std::exception & e) { EXPECT_EQUAL(vespalib::string("commit failed with code -2. server says: Exception during commit on " + name + " : Incomming serial number(1) must be bigger than the last one (3)."), e.what()); @@ -340,7 +340,7 @@ bool Test::fillDomainTest(TransLogClient::Session * s1, const vespalib::string & EXPECT_EQUAL(a.range().to(), 3u); Packet::Entry e; - vespalib::nbostream h(a.getHandle().c_str(), a.getHandle().size()); + vespalib::nbostream h(a.getHandle().data(), a.getHandle().size()); e.deserialize(h); e.deserialize(h); e.deserialize(h); @@ -358,13 +358,13 @@ void Test::fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_ Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&value, sizeof(value))); if ( ! p->add(e) ) { p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); p.reset(new Packet()); ASSERT_TRUE(p->add(e)); } } p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); } } @@ -382,13 +382,13 @@ Test::fillDomainTest(TransLogClient::Session * s1, Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&entryBuffer[0], entryBuffer.size())); if ( ! p->add(e) ) { p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); p.reset(new Packet()); ASSERT_TRUE(p->add(e)); } } p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); } } diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index a0e4b4884a9..74e48081e17 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -17,7 +17,7 @@ LOG_SETUP("translogstress"); -using document::ByteBuffer; +using vespalib::nbostream; using search::Runnable; using vespalib::Monitor; using vespalib::MonitorGuard; @@ -47,10 +47,10 @@ public: BufferGenerator(uint32_t minStrLen, uint32_t maxStrLen) : _rnd(), _minStrLen(minStrLen), _maxStrLen(maxStrLen) {} void setSeed(long seed) { _rnd.srand48(seed); } - ByteBuffer getRandomBuffer(); + nbostream getRandomBuffer(); }; -ByteBuffer +nbostream BufferGenerator::getRandomBuffer() { size_t len = _minStrLen + _rnd.lrand48() % (_maxStrLen - _minStrLen); @@ -59,9 +59,8 @@ BufferGenerator::getRandomBuffer() char c = 'a' + _rnd.lrand48() % ('z' - 'a' + 1); str.push_back(c); } - ByteBuffer buf(str.size() + 1); - buf.putBytes(str.c_str(), str.size() + 1); - buf.flip(); + nbostream buf(str.size() + 1); + buf.write(str.c_str(), str.size() + 1); return buf; } @@ -75,13 +74,13 @@ private: Rand48 _rnd; long _baseSeed; BufferGenerator _bufferGenerator; - const std::vector<document::ByteBuffer> * _buffers; - ByteBuffer _lastGeneratedBuffer; + const std::vector<nbostream> * _buffers; + nbostream _lastGeneratedBuffer; public: EntryGenerator(long baseSeed, const BufferGenerator & bufferGenerator) : - _rnd(), _baseSeed(baseSeed), _bufferGenerator(bufferGenerator), _buffers(NULL), - _lastGeneratedBuffer() {} + _rnd(), _baseSeed(baseSeed), _bufferGenerator(bufferGenerator), _buffers(nullptr), + _lastGeneratedBuffer(0) {} EntryGenerator(const EntryGenerator & rhs) : _rnd(), _baseSeed(rhs._baseSeed), _bufferGenerator(rhs._bufferGenerator), _buffers(rhs._buffers), _lastGeneratedBuffer(rhs._lastGeneratedBuffer) {} @@ -95,7 +94,7 @@ public: SerialNum getRandomSerialNum(SerialNum begin, SerialNum end); Packet::Entry getRandomEntry(SerialNum num); Rand48 & getRnd() { return _rnd; } - void setBuffers(const std::vector<ByteBuffer> & buffers) { + void setBuffers(const std::vector<nbostream> & buffers) { _buffers = &buffers; } }; @@ -118,12 +117,12 @@ EntryGenerator::getRandomEntry(SerialNum num) _rnd.srand48(_baseSeed + num); if (_buffers != NULL) { size_t i = _rnd.lrand48() % _buffers->size(); - const ByteBuffer& buffer = (*_buffers)[i]; - return Packet::Entry(num, 1024, ConstBufferRef(buffer.getBuffer(), buffer.getLength())); + const nbostream& buffer = (*_buffers)[i]; + return Packet::Entry(num, 1024, ConstBufferRef(buffer.data(), buffer.size())); } else { _bufferGenerator.setSeed(_baseSeed + num); _lastGeneratedBuffer = _bufferGenerator.getRandomBuffer(); - return Packet::Entry(num, 1024, ConstBufferRef(_lastGeneratedBuffer.getBuffer(), _lastGeneratedBuffer.getLength())); + return Packet::Entry(num, 1024, ConstBufferRef(_lastGeneratedBuffer.data(), _lastGeneratedBuffer.size())); } } @@ -226,7 +225,7 @@ FeederThread::commitPacket() { _packet.close(); const vespalib::nbostream& stream = _packet.getHandle(); - if (!_session->commit(ConstBufferRef(stream.c_str(), stream.size()))) { + if (!_session->commit(ConstBufferRef(stream.data(), stream.size()))) { throw std::runtime_error(vespalib::make_string ("FeederThread: Failed commiting %s", PacketPrinter::toStr(_packet).c_str())); } else { @@ -708,7 +707,7 @@ TransLogStress::Main() BufferGenerator bufferGenerator(_cfg.minStrLen, _cfg.maxStrLen); bufferGenerator.setSeed(_cfg.baseSeed); - std::vector<ByteBuffer> buffers; + std::vector<nbostream> buffers; for (uint32_t i = 0; i < _cfg.numPreGeneratedBuffers; ++i) { buffers.push_back(bufferGenerator.getRandomBuffer()); } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index a38cbe60e56..1e4bba95b4b 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -24,8 +24,7 @@ class SaveBits FA &_fa; public: - SaveBits(vespalib::ConstArrayRef<T> map, - FA &fa) + SaveBits(vespalib::ConstArrayRef<T> map, FA &fa) : _map(map), _fa(fa) { @@ -55,11 +54,9 @@ FlagAttributeT<B>::FlagAttributeT(const vespalib::string & baseFileName, const A template <typename B> AttributeVector::SearchContext::UP -FlagAttributeT<B>::getSearch(QueryTermSimple::UP qTerm, - const attribute::SearchContextParams & params) const +FlagAttributeT<B>::getSearch(QueryTermSimple::UP qTerm, const attribute::SearchContextParams &) const { - (void) params; - return AttributeVector::SearchContext::UP (new SearchContext(std::move(qTerm), *this)); + return std::make_unique<SearchContext>(std::move(qTerm), *this); } template <typename B> @@ -69,7 +66,7 @@ void FlagAttributeT<B>::clearOldValues(DocId doc) for (uint32_t i(0), m(this->get(doc, values)); i < m; i++) { BitVector * bv = _bitVectors[getOffset(values[i].value())]; if (bv != nullptr) { - bv->clearBit(doc); + bv->clearBitAndMaintainCount(doc); } } } @@ -130,10 +127,9 @@ void FlagAttributeT<B>::setNewValues(DocId doc, const std::vector<typename B::WT _bitVectorStore[offset] = BitVector::create(_bitVectorSize); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; - bv->invalidateCachedCount(); ensureGuardBit(*bv); } - bv->setBit(doc); + bv->setBitAndMaintainCount(doc); } } @@ -145,13 +141,12 @@ FlagAttributeT<B>::setNewBVValue(DocId doc, typename B::WType::ValueType value) BitVector * bv = _bitVectors[offset]; if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); - _bitVectorStore[offset] = BitVector::create(_bitVectorSize); + _bitVectorStore[offset] = BitVector::create(_bitVectorSize); _bitVectors[offset] = _bitVectorStore[offset].get(); bv = _bitVectors[offset]; - bv->invalidateCachedCount(); ensureGuardBit(*bv); } - bv->setBit(doc); + bv->setBitAndMaintainCount(doc); } @@ -193,8 +188,7 @@ template <typename B> void FlagAttributeT<B>::ensureGuardBit() { - for (uint32_t i = 0; i < _bitVectors.size(); ++i) { - BitVector * bv = _bitVectors[i]; + for (BitVector * bv : _bitVectors) { if (bv != nullptr) { ensureGuardBit(*bv); } @@ -205,8 +199,7 @@ template <typename B> void FlagAttributeT<B>::clearGuardBit(DocId doc) { - for (uint32_t i = 0; i < _bitVectors.size(); ++i) { - BitVector * bv = _bitVectors[i]; + for (BitVector * bv : _bitVectors) { if (bv != nullptr) { bv->clearBit(doc); // clear guard bit and start using this doc id } @@ -219,8 +212,7 @@ FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); - for (uint32_t i = 0; i < _bitVectors.size(); ++i) { - BitVector * bv = _bitVectors[i]; + for (BitVector * bv : _bitVectors) { if (bv != nullptr) { vespalib::GenerationHeldBase::UP hold(bv->grow(newSize)); ensureGuardBit(*bv); diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index 742d0cd8592..d3ce5f9af46 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -52,10 +52,10 @@ private: return _bitVectors[value + 128]; } - vespalib::GenerationHolder _bitVectorHolder; + vespalib::GenerationHolder _bitVectorHolder; std::vector<std::shared_ptr<BitVector> > _bitVectorStore; std::vector<BitVector *> _bitVectors; - uint32_t _bitVectorSize; + uint32_t _bitVectorSize; template <class SC> friend class FlagAttributeIteratorT; template <class SC> friend class FlagAttributeIteratorStrict; }; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 9c736a16069..2badb199934 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -246,9 +246,8 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) uint32_t typeId = getTypeId(iRef); assert(isBTree(typeId)); (void) typeId; - std::shared_ptr<GrowableBitVector> bvsp; vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); - bvsp.reset(new GrowableBitVector(_bvSize, _bvCapacity, genHolder)); + auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); AllocatedBitVector &bv = *bvsp.get(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; @@ -288,9 +287,8 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, { assert(!ref.valid()); RefType iRef(ref); - std::shared_ptr<GrowableBitVector> bvsp; vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); - bvsp.reset(new GrowableBitVector(_bvSize, _bvCapacity, genHolder)); + auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); AllocatedBitVector &bv = *bvsp.get(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; @@ -329,17 +327,17 @@ PostingStore<DataT>::apply(BitVector &bv, if (r != re && (a == ae || *r < a->_key)) { // remove assert(*r < bv.size()); - bv.slowClearBit(*r); + bv.clearBitAndMaintainCount(*r); ++r; } else { if (r != re && !(a->_key < *r)) { // update or add assert(a->_key < bv.size()); - bv.slowSetBit(a->_key); + bv.setBitAndMaintainCount(a->_key); ++r; } else { assert(a->_key < bv.size()); - bv.slowSetBit(a->_key); + bv.setBitAndMaintainCount(a->_key); } ++a; } diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 250182f924b..d350b479c66 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -60,25 +60,16 @@ SingleBoolAttribute::onCommit() { for (const auto & change : _changes) { if (change._type == ChangeBase::UPDATE) { std::atomic_thread_fence(std::memory_order_release); - if (change._data == 0) { - _bv.clearBit(change._doc); - } else { - _bv.setBit(change._doc); - } + setBit(change._doc, change._data != 0); } else if ((change._type >= ChangeBase::ADD) && (change._type <= ChangeBase::DIV)) { std::atomic_thread_fence(std::memory_order_release); int8_t val = applyArithmetic(getFast(change._doc), change); - if (val == 0) { - _bv.clearBit(change._doc); - } else { - _bv.setBit(change._doc); - } + setBit(change._doc, val != 0); } else if (change._type == ChangeBase::CLEARDOC) { std::atomic_thread_fence(std::memory_order_release); - _bv.clearBit(change._doc); + _bv.clearBitAndMaintainCount(change._doc); } } - _bv.invalidateCachedCount(); } std::atomic_thread_fence(std::memory_order_release); @@ -160,8 +151,7 @@ BitVectorSearchContext::createFilterIterator(fef::TermFieldMatchData * matchData } void -BitVectorSearchContext::fetchPostings(const queryeval::ExecuteInfo &execInfo) { - (void) execInfo; +BitVectorSearchContext::fetchPostings(const queryeval::ExecuteInfo &) { } std::unique_ptr<queryeval::SearchIterator> @@ -172,7 +162,9 @@ BitVectorSearchContext::createPostingIterator(fef::TermFieldMatchData *matchData unsigned int BitVectorSearchContext::approximateHits() const { return valid() - ? (_invert) ? (_bv.size() - _bv.countTrueBits()) : _bv.countTrueBits() + ? (_invert) + ? (_bv.size() - _bv.countTrueBits()) + : _bv.countTrueBits() : 0; } @@ -195,6 +187,7 @@ SingleBoolAttribute::onLoad() uint32_t numDocs = attrReader.getNextData(); _bv.extend(numDocs); ssize_t bytesRead = attrReader.getReader().read(_bv.getStart(), _bv.sizeBytes()); + _bv.invalidateCachedCount(); assert(bytesRead == _bv.sizeBytes()); setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 20ec0b6d077..78c297d55c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -91,9 +91,9 @@ public: const BitVector & getBitVector() const { return _bv; } void setBit(DocId doc, bool value) { if (value) { - _bv.setBit(doc); + _bv.setBitAndMaintainCount(doc); } else { - _bv.clearBit(doc); + _bv.clearBitAndMaintainCount(doc); } } protected: diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 7405688f4f7..ee6364235e3 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -134,17 +134,9 @@ public: void clearBit(Index idx) { _words[wordNum(idx)] &= ~ mask(idx); } - void flip(Index idx) { + void flipBit(Index idx) { _words[wordNum(idx)] ^= mask(idx); } - void slowSetBit(Index idx) { - if ( ! testBit(idx) ) { - setBit(idx); - if ( isValidCount() ) { - _numTrueBits++; - } - } - } void andWith(const BitVector &right); void orWith(const BitVector &right); @@ -171,12 +163,24 @@ public: */ void setInterval(Index start, Index end); - void slowClearBit(Index idx) { + /** + * Sets a bit and maintains count of number of bits set. + * @param idx + */ + void setBitAndMaintainCount(Index idx) { + if ( ! testBit(idx) ) { + setBit(idx); + incNumBits(); + } + } + /** + * Clears a bit and maintains count of number of bits set. + * @param idx + */ + void clearBitAndMaintainCount(Index idx) { if (testBit(idx)) { clearBit(idx); - if ( isValidCount() ) { - _numTrueBits--; - } + decNumBits(); } } @@ -279,6 +283,16 @@ private: static size_t numActiveWords(Index start, Index end) { return (numWords(end) - wordNum(start)); } static Index invalidCount() { return std::numeric_limits<Index>::max(); } void setGuardBit() { setBit(size()); } + void incNumBits() { + if ( isValidCount() ) { + _numTrueBits++; + } + } + void decNumBits() { + if ( isValidCount() ) { + _numTrueBits--; + } + } VESPA_DLL_LOCAL void repairEnds(); VESPA_DLL_LOCAL static Index internalCount(const Word *tarr, size_t sz); Index count() const; diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 4707e8001a8..c7976fd15ae 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -89,13 +89,13 @@ Chunk::getLid(uint32_t lid) const if (it->getLid() == lid) { #if 1 uint32_t bLid(0), bLen(0); - vespalib::nbostream is(getData().c_str()+it->getOffset(), it->size()); + vespalib::nbostream is(getData().data() + it->getOffset(), it->size()); is >> bLid >> bLen; assert(bLid == lid); assert(bLen == it->netSize()); assert((bLen + 2*sizeof(uint32_t)) == it->size()); #endif - buf = vespalib::ConstBufferRef(getData().c_str() + it->getNetOffset(), it->netSize()); + buf = vespalib::ConstBufferRef(getData().data() + it->getNetOffset(), it->netSize()); } } return buf; diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 4d76b3fea25..a5bed4c33ce 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -33,7 +33,7 @@ ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const const size_t oldPos(compressed.getDataLen()); compressed.writeInt8(compression.type); compressed.writeInt32(os.size()); - CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.c_str(), os.size()), compressed, false)); + CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.data(), os.size()), compressed, false)); if (compression.type != type) { compressed.getData()[oldPos] = type; } diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 994df3237f2..e8504480b7d 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -66,7 +66,7 @@ BlobSet::get(uint32_t lid) const ConstBufferRef buf; for (LidPosition pos : _positions) { if (pos.lid() == lid) { - buf = ConstBufferRef(_buffer.c_str() + pos.offset(), pos.size()); + buf = ConstBufferRef(_buffer.data() + pos.offset(), pos.size()); break; } } diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index eb035ac2a2c..8a06794ee35 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -60,7 +60,7 @@ public: void remove(uint32_t lid); const Positions & getPositions() const { return _positions; } vespalib::ConstBufferRef get(uint32_t lid) const; - vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.c_str(), _buffer.size()); } + vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.data(), _buffer.size()); } private: Positions _positions; vespalib::nbostream _buffer; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index e802ec0a326..70295d81a93 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -867,13 +867,13 @@ WriteableFileChunk::unconditionallyFlushPendingChunks(const vespalib::LockGuard _pendingDat -= pc.getDataLen(); lastSerial = pc.getLastSerial(); const nbostream &os2(pc.getSerializedIdx()); - os.write(os2.c_str(), os2.size()); + os.write(os2.data(), os2.size()); } } vespalib::system_time timeStamp(vespalib::system_clock::now()); auto idxFile = openIdx(); idxFile->SetPosition(idxFile->GetSize()); - ssize_t wlen = idxFile->Write2(os.c_str(), os.size()); + ssize_t wlen = idxFile->Write2(os.data(), os.size()); updateCurrentDiskFootprint(); if (wlen != static_cast<ssize_t>(os.size())) { diff --git a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp index 0eb85cba4ba..b770b9ffc9d 100644 --- a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp @@ -489,7 +489,7 @@ bool CatFunctionNode::onExecute() const getArg(i).execute(); getArg(i).getResult().serialize(nos); } - static_cast<RawResultNode &>(updateResult()).setBuffer(os.c_str(), os.size()); + static_cast<RawResultNode &>(updateResult()).setBuffer(os.data(), os.size()); return true; } @@ -520,14 +520,14 @@ bool XorBitFunctionNode::internalExecute(const nbostream & os) const { const size_t numBytes(_tmpXor.size()); memset(&_tmpXor[0], 0, numBytes); - const char * s(os.c_str()); + const char * s(os.data()); for (size_t i(0), m(os.size()/numBytes); i < m; i++) { for (size_t j(0), k(numBytes); j < k; j++) { _tmpXor[j] ^= s[j + k*i]; } } for (size_t i((os.size()/numBytes)*numBytes); i < os.size(); i++) { - _tmpXor[i%numBytes] = os.c_str()[i]; + _tmpXor[i%numBytes] = os.data()[i]; } static_cast<RawResultNode &>(updateResult()).setBuffer(&_tmpXor[0], numBytes); return true; @@ -537,7 +537,7 @@ bool MD5BitFunctionNode::internalExecute(const nbostream & os) const { const unsigned int MD5_DIGEST_LENGTH = 16; unsigned char md5ScratchPad[MD5_DIGEST_LENGTH]; - fastc_md5sum(os.c_str(), os.size(), md5ScratchPad); + fastc_md5sum(os.data(), os.size(), md5ScratchPad); static_cast<RawResultNode &>(updateResult()).setBuffer(md5ScratchPad, std::min(sizeof(md5ScratchPad), getNumBytes())); return true; } diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp index 4e522f27ce2..ff41396f66b 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp @@ -3,7 +3,6 @@ #include "generic_tensor_store.h" #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/serializable.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/objects/nbostream.h> @@ -15,9 +14,7 @@ using search::datastore::Handle; using vespalib::tensor::Tensor; using vespalib::tensor::TypedBinaryFormat; -namespace search { - -namespace tensor { +namespace search::tensor { constexpr size_t MIN_BUFFER_ARRAYS = 1024; @@ -118,6 +115,4 @@ GenericTensorStore::setTensor(const Tensor &tensor) return raw.ref; } -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index a84e27b2e53..a5eaa61af12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -37,7 +37,7 @@ Packet::Packet(const void * buf, size_t sz) : _limit(sz), _buf(static_cast<const char *>(buf), sz) { - nbostream_longlivedbuf os(_buf.c_str(), sz); + nbostream_longlivedbuf os(_buf.data(), sz); while ( os.size() > 0 ) { Entry e; e.deserialize(os); @@ -55,7 +55,7 @@ bool Packet::merge(const Packet & packet) if (retval) { _count += packet._count; _range.to(packet._range.to()); - _buf.write(packet.getHandle().c_str(), packet.getHandle().size()); + _buf.write(packet.getHandle().data(), packet.getHandle().size()); } return retval; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index fc9518ccf1b..5a64d829183 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -282,7 +282,7 @@ void waitPendingSync(vespalib::Monitor &syncMonitor, bool &pendingSync) void Domain::commit(const Packet & packet) { DomainPart::SP dp(_parts.rbegin()->second); - vespalib::nbostream_longlivedbuf is(packet.getHandle().c_str(), packet.getHandle().size()); + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); if (dp->byteSize() > _domainPartSize) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index d2838711a51..8a6e833bd1f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -403,7 +403,7 @@ void DomainPart::commit(SerialNum firstSerial, const Packet &packet) { int64_t firstPos(_transLog->GetPosition()); - nbostream_longlivedbuf h(packet.getHandle().c_str(), packet.getHandle().size()); + nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); if (_range.from() == 0) { _range.from(firstSerial); } @@ -495,7 +495,7 @@ DomainPart::visit(SerialNumRange &r, Packet &packet) } } else { const nbostream & tmp = start->second.getHandle(); - nbostream_longlivedbuf h(tmp.c_str(), tmp.size()); + nbostream_longlivedbuf h(tmp.data(), tmp.size()); LOG(debug, "Visit partial[%" PRIu64 ", %" PRIu64 "] (%zd, %zd, %zd)", start->second.range().from(), start->second.range().to(), h.rp(), h.size(), h.capacity()); Packet newPacket(h.size()); @@ -585,13 +585,13 @@ DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry) size_t start(os.size()); entry.serialize(os); size_t end(os.size()); - crc = calcCrc(_defaultCrc, os.c_str()+start, end - start); + crc = calcCrc(_defaultCrc, os.data() + start, end - start); os << crc; size_t osSize = os.size(); assert(osSize == len + sizeof(len) + sizeof(uint8_t)); LockGuard guard(_writeLock); - if ( ! file.CheckedWrite(os.c_str(), osSize) ) { + if ( ! file.CheckedWrite(os.data(), osSize) ) { throw runtime_error(handleWriteError("Failed writing the entry.", file, lastKnownGoodPos, entry, end - start)); } _writtenSerial = entry.serial(); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 37903bc21f5..a3528c4f615 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -366,7 +366,7 @@ public: req->SetMethodName("visitCallback"); req->GetParams()->AddString(domain.c_str()); req->GetParams()->AddInt32(id); - req->GetParams()->AddData(packet.getHandle().c_str(), packet.getHandle().size()); + req->GetParams()->AddData(packet.getHandle().data(), packet.getHandle().size()); return send(req); } diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 97b6cc344e1..cefa8ab2f51 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -19,11 +19,16 @@ import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; import java.security.GeneralSecurityException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Random; import static com.yahoo.security.Extension.SUBJECT_ALTERNATIVE_NAMES; import static java.util.stream.Collectors.toList; @@ -140,4 +145,20 @@ public class X509CertificateUtils { } } + public static boolean privateKeyMatchesPublicKey(PrivateKey privateKey, PublicKey publicKey) { + byte[] someRandomData = new byte[64]; + new Random().nextBytes(someRandomData); + + Signature signer = SignatureUtils.createSigner(privateKey); + Signature verifier = SignatureUtils.createVerifier(publicKey); + try { + signer.update(someRandomData); + verifier.update(someRandomData); + byte[] signature = signer.sign(); + return verifier.verify(signature); + } catch (SignatureException e) { + throw new RuntimeException(e); + } + } + } diff --git a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java index 76a93028efe..b4eca8328c1 100644 --- a/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/X509CertificateUtilsTest.java @@ -17,7 +17,9 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author bjorncs @@ -71,4 +73,18 @@ public class X509CertificateUtilsTest { assertThat(sans.size(), is(1)); assertThat(sans.get(0), equalTo(san)); } + + @Test + public void verifies_matching_cert_and_key() { + KeyPair ecKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + KeyPair ecKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256); + KeyPair rsaKeypairA = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); + KeyPair rsaKeypairB = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 1024); + + assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairA.getPublic())); + assertTrue(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairA.getPublic())); + + assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(ecKeypairA.getPrivate(), ecKeypairB.getPublic())); + assertFalse(X509CertificateUtils.privateKeyMatchesPublicKey(rsaKeypairA.getPrivate(), rsaKeypairB.getPublic())); + } }
\ No newline at end of file diff --git a/staging_vespalib/src/tests/objects/identifiable_test.cpp b/staging_vespalib/src/tests/objects/identifiable_test.cpp index b952ad18333..2b4d50cc786 100644 --- a/staging_vespalib/src/tests/objects/identifiable_test.cpp +++ b/staging_vespalib/src/tests/objects/identifiable_test.cpp @@ -161,40 +161,40 @@ void IdentifiableTest::testNboStream() EXPECT_EQUAL(nbostream::ok, s.state()); EXPECT_EQUAL(10u, s.size()); EXPECT_EQUAL(16u, s.capacity()); - EXPECT_EQUAL(0, strncmp(s.c_str()+4, "abcdef", 6)); + EXPECT_EQUAL(0, strncmp(s.data() + 4, "abcdef", 6)); } { nbostream s(8); EXPECT_EQUAL(0u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - const char * prev = s.c_str(); + const char * prev = s.data(); s << "ABCD"; EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); s << "A long string that will cause resizing"; EXPECT_EQUAL(50u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_NOT_EQUAL(prev, s.data()); } { nbostream s(8); EXPECT_EQUAL(0u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - const char * prev = s.c_str(); + const char * prev = s.data(); s << "ABCD"; EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); s.reserve(50); - EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_NOT_EQUAL(prev, s.data()); EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - prev = s.c_str(); + prev = s.data(); s << "A long string that will cause resizing"; EXPECT_EQUAL(50u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); } { nbostream s; diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp index 6cc2af1fc90..2465d5f9d9b 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -244,7 +244,7 @@ int Identifiable::onCmp(const Identifiable& b) const nbs << b; size_t minLength(std::min(as.size(), bs.size())); if (minLength > 0) { - diff = memcmp(as.c_str(), bs.c_str(), minLength); + diff = memcmp(as.data(), bs.data(), minLength); } if (diff == 0) { diff = as.size() - bs.size(); diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 57e88873e81..a10cad70579 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -27,7 +27,7 @@ GrowableByteBuffer::allocate(uint32_t len) } void -GrowableByteBuffer::putBytes(const char* buffer, uint32_t length) +GrowableByteBuffer::putBytes(const void * buffer, uint32_t length) { char* buf = allocate(length); memcpy(buf, buffer, length); diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h index d32afeeabee..afbde04fb9b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -45,7 +45,7 @@ public: /** Adds the given buffer to this buffer. */ - void putBytes(const char* buffer, uint32_t length); + void putBytes(const void * buffer, uint32_t length); /** Adds a short to the buffer. diff --git a/storage/src/tests/common/message_sender_stub.cpp b/storage/src/tests/common/message_sender_stub.cpp index c127f9071e5..a82d45b0b99 100644 --- a/storage/src/tests/common/message_sender_stub.cpp +++ b/storage/src/tests/common/message_sender_stub.cpp @@ -22,9 +22,7 @@ MessageSenderStub::getLastCommand(bool verbose) const } std::string -MessageSenderStub::dumpMessage(const api::StorageMessage& msg, - bool includeAddress, - bool verbose) const +MessageSenderStub::dumpMessage(const api::StorageMessage& msg, bool includeAddress, bool verbose) const { std::ostringstream ost; @@ -67,9 +65,7 @@ MessageSenderStub::getLastReply(bool verbose) const throw std::logic_error("Expected reply where there was none"); } - return dumpMessage(*replies.back(), - true, - verbose); + return dumpMessage(*replies.back(),true, verbose); } diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 64306fa7c24..4576f8a08f8 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1501,7 +1501,7 @@ TEST_F(FileStorManagerTest, visiting) { for (uint32_t i=3; i<docCount; ++i) { auto reply = std::dynamic_pointer_cast<api::BucketInfoReply>(top.getReply(i)); ASSERT_TRUE(reply.get()); - ASSERT_TRUE(reply->getResult().success()) << reply->getResult().toString(); + ASSERT_TRUE(reply->getResult().success()) << reply->getResult(); info = reply->getBucketInfo(); } diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index b7eb7fee3ec..20934d04eaa 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -20,6 +20,7 @@ #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/objects/nbostream.h> #include <gmock/gmock.h> #include <optional> #include <thread> @@ -337,7 +338,7 @@ int getTotalSerializedSize(const std::vector<document::Document::SP>& docs) { int total = 0; for (size_t i = 0; i < docs.size(); ++i) { - total += int(docs[i]->serialize()->getLength()); + total += int(docs[i]->serialize().size()); } return total; } diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index e92e2d4c3bf..1a4dc61a3ce 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -14,7 +14,7 @@ ReadBucketList::ReadBucketList(BucketSpace bucketSpace, spi::PartitionId partiti _partition(partition) { } -ReadBucketList::~ReadBucketList() { } +ReadBucketList::~ReadBucketList() = default; document::Bucket ReadBucketList::getBucket() const @@ -38,7 +38,7 @@ ReadBucketListReply::ReadBucketListReply(const ReadBucketList& cmd) _partition(cmd.getPartition()) { } -ReadBucketListReply::~ReadBucketListReply() { } +ReadBucketListReply::~ReadBucketListReply() = default; document::Bucket ReadBucketListReply::getBucket() const @@ -66,7 +66,7 @@ ReadBucketInfo::ReadBucketInfo(const document::Bucket &bucket) _bucket(bucket) { } -ReadBucketInfo::~ReadBucketInfo() { } +ReadBucketInfo::~ReadBucketInfo() = default; void ReadBucketInfo::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -92,7 +92,7 @@ ReadBucketInfoReply::ReadBucketInfoReply(const ReadBucketInfo& cmd) _bucket(cmd.getBucket()) { } -ReadBucketInfoReply::~ReadBucketInfoReply() { } +ReadBucketInfoReply::~ReadBucketInfoReply() = default; void ReadBucketInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "ReadBucketInfoReply()"; @@ -117,7 +117,7 @@ RepairBucketCommand::RepairBucketCommand(const document::Bucket &bucket, uint16_ setPriority(LOW); } -RepairBucketCommand::~RepairBucketCommand() { } +RepairBucketCommand::~RepairBucketCommand() = default; void RepairBucketCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -153,7 +153,7 @@ RepairBucketReply::RepairBucketReply(const RepairBucketCommand& cmd, const api:: _altered(false) { } -RepairBucketReply::~RepairBucketReply() { } +RepairBucketReply::~RepairBucketReply() = default; void RepairBucketReply::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -180,7 +180,7 @@ BucketDiskMoveCommand::BucketDiskMoveCommand(const document::Bucket &bucket, setPriority(LOW); } -BucketDiskMoveCommand::~BucketDiskMoveCommand() { } +BucketDiskMoveCommand::~BucketDiskMoveCommand() = default; void BucketDiskMoveCommand::setBucketId(const document::BucketId& id) @@ -208,14 +208,13 @@ BucketDiskMoveReply::BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, _dstDisk(cmd.getDstDisk()) { } -BucketDiskMoveReply::~BucketDiskMoveReply() { } +BucketDiskMoveReply::~BucketDiskMoveReply() = default; void BucketDiskMoveReply::print(std::ostream& out, bool, const std::string&) const { out << "BucketDiskMoveReply(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ", " << _bucketInfo << ", " - << getResult() << ")"; + << ", target " << _dstDisk << ", " << _bucketInfo << ", " << getResult() << ")"; } std::unique_ptr<api::StorageReply> @@ -236,7 +235,7 @@ InternalBucketJoinCommand::InternalBucketJoinCommand(const document::Bucket &buc // them higher than getting more bucket info lists. } -InternalBucketJoinCommand::~InternalBucketJoinCommand() { } +InternalBucketJoinCommand::~InternalBucketJoinCommand() = default; void InternalBucketJoinCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -255,7 +254,7 @@ InternalBucketJoinReply::InternalBucketJoinReply(const InternalBucketJoinCommand _bucketInfo(info) { } -InternalBucketJoinReply::~InternalBucketJoinReply() { } +InternalBucketJoinReply::~InternalBucketJoinReply() = default; void InternalBucketJoinReply::print(std::ostream& out, bool verbose, const std::string& indent) const diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index f73eb3ea36d..431c90b27f2 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -4,6 +4,7 @@ #include "bucketmessages.h" #include <vespa/vespalib/util/backtrace.h> #include <sstream> +#include <cassert> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".application.link"); @@ -141,7 +142,7 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) sendUp(reply); } } else { - ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); + ost << " Return code: " << static_cast<const StorageReply&>(*msg).getResult(); LOGBP(warning, "%s", ost.str().c_str()); } } else if (!_down->onDown(msg)) { @@ -181,7 +182,7 @@ void StorageLink::sendUp(const shared_ptr<StorageMessage> & msg) sendDown(reply); } } else { - ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); + ost << " Return code: " << static_cast<const StorageReply&>(*msg).getResult(); LOGBP(warning, "%s", ost.str().c_str()); } } else if (!_up->onUp(msg)) { diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index e143f4d8570..15a57c1e7ee 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -2,6 +2,7 @@ #include "simplemaintenancescanner.h" #include <vespa/storage/distributor/distributor_bucket_space.h> #include <ostream> +#include <cassert> namespace storage::distributor { @@ -16,10 +17,10 @@ SimpleMaintenanceScanner::SimpleMaintenanceScanner(BucketPriorityDatabase& bucke { } -SimpleMaintenanceScanner::~SimpleMaintenanceScanner() {} +SimpleMaintenanceScanner::~SimpleMaintenanceScanner() = default; -SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() {} -SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() {} +SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() = default; +SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() = default; SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats(const PendingMaintenanceStats &) = default; SimpleMaintenanceScanner::PendingMaintenanceStats & SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (const PendingMaintenanceStats &) = default; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp index 8e6494a588d..60c1137bd6d 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp @@ -8,8 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.statbucket"); -namespace storage { -namespace distributor { +namespace storage::distributor { StatBucketOperation::StatBucketOperation( [[maybe_unused]] DistributorComponent& manager, @@ -21,7 +20,7 @@ StatBucketOperation::StatBucketOperation( { } -StatBucketOperation::~StatBucketOperation() {} +StatBucketOperation::~StatBucketOperation() = default; void StatBucketOperation::onClose(DistributorMessageSender& sender) @@ -36,8 +35,7 @@ StatBucketOperation::onStart(DistributorMessageSender& sender) { std::vector<uint16_t> nodes; - BucketDatabase::Entry entry( - _bucketSpace.getBucketDatabase().get(_command->getBucketId())); + BucketDatabase::Entry entry(_bucketSpace.getBucketDatabase().get(_command->getBucketId())); if (entry.valid()) { nodes = entry->getNodes(); @@ -103,5 +101,4 @@ StatBucketOperation::onReceive(DistributorMessageSender& sender, const std::shar } } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp b/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp index abd9778d72c..9e3230a0f34 100644 --- a/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp +++ b/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "throttlingoperationstarter.h" +#include <cassert> namespace storage::distributor { @@ -10,8 +11,7 @@ ThrottlingOperationStarter::ThrottlingOperation::~ThrottlingOperation() } bool -ThrottlingOperationStarter::canStart(uint32_t currentOperationCount, - Priority priority) const +ThrottlingOperationStarter::canStart(uint32_t currentOperationCount, Priority priority) const { uint32_t variablePending(_maxPending - _minPending); uint32_t maxPendingForPri(_minPending + variablePending*((255.0 - priority) / 255.0)); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index bfc89a70a85..65d4035a3dd 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -36,11 +36,8 @@ namespace api { class StorageReply; } -class BucketMergeTest; -class DiskInfo; struct FileStorManagerTest; class ReadBucketList; -class ModifiedBucketCheckerThread; class BucketOwnershipNotifier; class AbortBucketOperationsCommand; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 37e1d818bb8..e5e358cbb60 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -506,18 +506,11 @@ MergeHandler::fetchLocalData( assert(doc != 0); assertContainedInBucket(doc->getId(), bucket, idFactory); e._docName = doc->getId().toString(); - { - vespalib::nbostream stream; - doc->serializeHeader(stream); - e._headerBlob.resize(stream.size()); - memcpy(&e._headerBlob[0], stream.peek(), stream.size()); - } - { - vespalib::nbostream stream; - doc->serializeBody(stream); - e._bodyBlob.resize(stream.size()); - memcpy(&e._bodyBlob[0], stream.peek(), stream.size()); - } + vespalib::nbostream stream; + doc->serialize(stream); + e._headerBlob.resize(stream.size()); + memcpy(&e._headerBlob[0], stream.peek(), stream.size()); + e._bodyBlob.clear(); } else { const DocumentId* docId = docEntry.getDocumentId(); assert(docId != 0); @@ -556,11 +549,11 @@ MergeHandler::deserializeDiffDocument( const api::ApplyBucketDiffCommand::Entry& e, const document::DocumentTypeRepo& repo) const { - Document::UP doc(new Document); - using document::ByteBuffer; - ByteBuffer hbuf(&e._headerBlob[0], e._headerBlob.size()); + auto doc = std::make_unique<Document>(); + vespalib::nbostream hbuf(&e._headerBlob[0], e._headerBlob.size()); if (e._bodyBlob.size() > 0) { - ByteBuffer bbuf(&e._bodyBlob[0], e._bodyBlob.size()); + // TODO Remove this branch and add warning on error. + vespalib::nbostream bbuf(&e._bodyBlob[0], e._bodyBlob.size()); doc->deserialize(repo, hbuf, bbuf); } else { doc->deserialize(repo, hbuf); diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index 6fc9795993e..b6bceabf7a1 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -77,7 +77,7 @@ OpsLogger::onPutReply(const std::shared_ptr<api::PutReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tPUT\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -94,7 +94,7 @@ OpsLogger::onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tUPDATE\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -111,7 +111,7 @@ OpsLogger::onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tREMOVE\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -128,7 +128,7 @@ OpsLogger::onGetReply(const std::shared_ptr<api::GetReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tGET\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index f0e64f0dfd1..f3240f0663b 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -16,37 +16,6 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default; -DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) - : metrics::MetricSet("document_serialization", {{"docserialization"}}, - "Counts of document serialization of various types", owner), - usedCachedSerializationCount( - "cached_serialization_count", {{"docserialization"}}, - "Number of times we didn't need to serialize the document as " - "we already had serialized version cached", this), - compressedDocumentCount( - "compressed_serialization_count", {{"docserialization"}}, - "Number of times we compressed document when serializing", - this), - compressionDidntHelpCount( - "compressed_didnthelp_count", {{"docserialization"}}, - "Number of times we compressed document when serializing, but " - "the compressed version was bigger, so it was dumped", this), - uncompressableCount( - "uncompressable_serialization_count", {{"docserialization"}}, - "Number of times we didn't attempt compression as document " - "had already been tagged uncompressable", this), - serializedUncompressed( - "uncompressed_serialization_count", {{"docserialization"}}, - "Number of times we serialized a document uncompressed", this), - inputWronglySerialized( - "input_wrongly_serialized_count", {{"docserialization"}}, - "Number of times we reserialized a document because the " - "compression it had in cache did not match what was configured", - this) -{} - -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default; - StorageMetricSet::StorageMetricSet() : metrics::MetricSet("server", {{"memory"}}, "Metrics for VDS applications"), @@ -54,28 +23,12 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - documentSerialization(this), tls_metrics(this) {} StorageMetricSet::~StorageMetricSet() = default; void StorageMetricSet::updateMetrics() { - document::SerializableArray::Statistics stats( - document::SerializableArray::getStatistics()); - - documentSerialization.usedCachedSerializationCount.set( - stats._usedCachedSerializationCount); - documentSerialization.compressedDocumentCount.set( - stats._compressedDocumentCount); - documentSerialization.compressionDidntHelpCount.set( - stats._compressionDidntHelpCount); - documentSerialization.uncompressableCount.set( - stats._uncompressableCount); - documentSerialization.serializedUncompressed.set( - stats._serializedUncompressed); - documentSerialization.inputWronglySerialized.set( - stats._inputWronglySerialized); // Delta snapshotting is destructive, so if an explicit snapshot is triggered // (instead of just regular periodic snapshots), some events will effectively diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index e9378010540..49795c63324 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -3,7 +3,6 @@ #pragma once #include "tls_statistics_metrics_wrapper.h" - #include <vespa/metrics/metrics.h> namespace storage { @@ -17,21 +16,8 @@ public: metrics::LongValueMetric highpri; metrics::LongValueMetric veryhighpri; - MessageMemoryUseMetricSet(metrics::MetricSet* owner); - ~MessageMemoryUseMetricSet(); -}; - -struct DocumentSerializationMetricSet : public metrics::MetricSet -{ - metrics::LongCountMetric usedCachedSerializationCount; - metrics::LongCountMetric compressedDocumentCount; - metrics::LongCountMetric compressionDidntHelpCount; - metrics::LongCountMetric uncompressableCount; - metrics::LongCountMetric serializedUncompressed; - metrics::LongCountMetric inputWronglySerialized; - - DocumentSerializationMetricSet(metrics::MetricSet* owner); - ~DocumentSerializationMetricSet(); + explicit MessageMemoryUseMetricSet(metrics::MetricSet* owner); + ~MessageMemoryUseMetricSet() override; }; struct StorageMetricSet : public metrics::MetricSet @@ -39,12 +25,11 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse; MessageMemoryUseMetricSet memoryUse_messages; metrics::LongValueMetric memoryUse_visiting; - DocumentSerializationMetricSet documentSerialization; TlsStatisticsMetricsWrapper tls_metrics; StorageMetricSet(); - ~StorageMetricSet(); + ~StorageMetricSet() override; void updateMetrics(); }; diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index 7c69a232af0..80e74e890a1 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -2,7 +2,7 @@ #include "recoveryvisitor.h" - +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -36,10 +36,9 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, { vespalib::LockGuard guard(_mutex); - LOG(debug, "Visitor %s handling block of %zu documents.", - _id.c_str(), entries.size()); + LOG(debug, "Visitor %s handling block of %zu documents.", _id.c_str(), entries.size()); - documentapi::DocumentListMessage* cmd = NULL; + documentapi::DocumentListMessage* cmd = nullptr; { CommandMap::iterator iter = _activeCommands.find(bid); @@ -71,7 +70,7 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, } } - hitCounter.addHit(doc->getId(), doc->serialize()->getLength()); + hitCounter.addHit(doc->getId(), doc->serialize().size()); int64_t timestamp = doc->getLastModified(); cmd->getDocuments().push_back(documentapi::DocumentListMessage::Entry( diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index e68a8fdbc8c..1da2acfed9c 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -12,16 +12,13 @@ #include "visitor.h" #include <vespa/storageapi/message/datagram.h> -namespace documentapi { -class DocumentListMessage; -} +namespace documentapi { class DocumentListMessage; } namespace storage { class RecoveryVisitor : public Visitor { public: - RecoveryVisitor(StorageComponent&, - const vdslib::Parameters& params); + RecoveryVisitor(StorageComponent&, const vdslib::Parameters& params); private: void handleDocuments(const document::BucketId& bucketId, @@ -43,12 +40,11 @@ struct RecoveryVisitorFactory : public VisitorFactory { VisitorEnvironment::UP makeVisitorEnvironment(StorageComponent&) override { - return VisitorEnvironment::UP(new VisitorEnvironment); + return std::make_unique<VisitorEnvironment>(); }; Visitor* - makeVisitor(StorageComponent& c, VisitorEnvironment&, - const vdslib::Parameters& params) override + makeVisitor(StorageComponent& c, VisitorEnvironment&, const vdslib::Parameters& params) override { return new RecoveryVisitor(c, params); } diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 4b213dff1d5..bdd066e8a4a 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -44,17 +44,12 @@ Visitor::HitCounter::addHit(const document::DocumentId& , uint32_t size) } void -Visitor::HitCounter::updateVisitorStatistics( - vdslib::VisitorStatistics& statistics) +Visitor::HitCounter::updateVisitorStatistics(vdslib::VisitorStatistics& statistics) { - statistics.setDocumentsReturned( - statistics.getDocumentsReturned() + _firstPassHits); - statistics.setBytesReturned( - statistics.getBytesReturned() + _firstPassBytes); - statistics.setSecondPassDocumentsReturned( - statistics.getSecondPassDocumentsReturned() + _secondPassHits); - statistics.setSecondPassBytesReturned( - statistics.getSecondPassBytesReturned() + _secondPassBytes); + statistics.setDocumentsReturned(statistics.getDocumentsReturned() + _firstPassHits); + statistics.setBytesReturned(statistics.getBytesReturned() + _firstPassBytes); + statistics.setSecondPassDocumentsReturned(statistics.getSecondPassDocumentsReturned() + _secondPassHits); + statistics.setSecondPassBytesReturned(statistics.getSecondPassBytesReturned() + _secondPassBytes); } Visitor::VisitorTarget::MessageMeta::MessageMeta( @@ -68,8 +63,7 @@ Visitor::VisitorTarget::MessageMeta::MessageMeta( { } -Visitor::VisitorTarget::MessageMeta::MessageMeta( - Visitor::VisitorTarget::MessageMeta&& rhs) noexcept +Visitor::VisitorTarget::MessageMeta::MessageMeta(Visitor::VisitorTarget::MessageMeta&& rhs) noexcept : messageId(rhs.messageId), retryCount(rhs.retryCount), memoryUsage(rhs.memoryUsage), @@ -78,9 +72,7 @@ Visitor::VisitorTarget::MessageMeta::MessageMeta( { } -Visitor::VisitorTarget::MessageMeta::~MessageMeta() -{ -} +Visitor::VisitorTarget::MessageMeta::~MessageMeta() = default; Visitor::VisitorTarget::MessageMeta& Visitor::VisitorTarget::MessageMeta::operator=( diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index b90153c9517..0cfd2160497 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -24,8 +24,7 @@ ProtocolSerialization4_2::ProtocolSerialization4_2( { } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::GetCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -41,14 +40,12 @@ ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); bool headerOnly(SH::getBoolean(buf)); - api::GetCommand::UP msg( - new api::GetCommand(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp)); + auto msg = std::make_unique<api::GetCommand>(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp); onDecodeCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::RemoveCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -62,13 +59,12 @@ ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const document::DocumentId did(SH::getString(buf)); document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); - api::RemoveCommand::UP msg(new api::RemoveCommand(bucket, did, timestamp)); + auto msg = std::make_unique<api::RemoveCommand>(bucket, did, timestamp); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::RevertCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RevertCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putInt(msg.getRevertTokens().size()); @@ -86,13 +82,12 @@ ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const for (uint32_t i=0, n=tokens.size(); i<n; ++i) { tokens[i] = SH::getLong(buf); } - api::RevertCommand::UP msg(new api::RevertCommand(bucket, tokens)); + auto msg = std::make_unique<api::RevertCommand>(bucket, tokens); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::CreateBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); @@ -102,13 +97,12 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); + auto msg = std::make_unique<api::CreateBucketCommand>(bucket); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::MergeBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -135,14 +129,12 @@ ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } api::Timestamp timestamp(SH::getLong(buf)); - api::MergeBucketCommand::UP msg( - new api::MergeBucketCommand(bucket, nodes, timestamp)); + auto msg = std::make_unique<api::MergeBucketCommand>(bucket, nodes, timestamp); onDecodeCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::GetBucketDiffCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -174,8 +166,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } api::Timestamp timestamp = SH::getLong(buf); - api::GetBucketDiffCommand::UP msg( - new api::GetBucketDiffCommand(bucket, nodes, timestamp)); + auto msg = std::make_unique<api::GetBucketDiffCommand>(bucket, nodes, timestamp); std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -190,8 +181,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -201,18 +191,15 @@ void ProtocolSerialization4_2::onEncode( buf.putBoolean(nodes[i].sourceOnly); } buf.putInt(msg.getMaxBufferSize()); - const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( - msg.getDiff()); + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); for (uint32_t i=0; i<entries.size(); ++i) { onEncodeDiffEntry(buf, entries[i]._entry); buf.putString(entries[i]._docName); buf.putInt(entries[i]._headerBlob.size()); - buf.putBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); buf.putInt(entries[i]._bodyBlob.size()); - buf.putBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onEncodeBucketInfoCommand(buf, msg); } @@ -231,8 +218,7 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } uint32_t maxBufferSize(SH::getInt(buf)); - api::ApplyBucketDiffCommand::UP msg( - new api::ApplyBucketDiffCommand(bucket, nodes, maxBufferSize)); + auto msg = std::make_unique<api::ApplyBucketDiffCommand>(bucket, nodes, maxBufferSize); std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -248,15 +234,13 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const buf.incPos(headerSize); } entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); uint32_t bodySize = SH::getInt(buf); if (bodySize > buf.getRemaining()) { buf.incPos(bodySize); } entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onDecodeBucketInfoCommand(buf, *msg); return msg; @@ -274,11 +258,9 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RequestBucketInfoReply } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const { - api::RequestBucketInfoReply::UP msg(new api::RequestBucketInfoReply( - static_cast<const api::RequestBucketInfoCommand&>(cmd))); + auto msg = std::make_unique<api::RequestBucketInfoReply>(static_cast<const api::RequestBucketInfoCommand&>(cmd)); api::RequestBucketInfoReply::EntryVector & entries(msg->getBucketInfo()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -294,8 +276,7 @@ ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { putBucket(msg.getBucket(), buf); putBucketInfo(msg.getBucketInfo(), buf); @@ -307,30 +288,25 @@ ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); api::BucketInfo info(getBucketInfo(buf)); - api::NotifyBucketChangeCommand::UP msg( - new api::NotifyBucketChangeCommand(bucket, info)); + auto msg = std::make_unique<api::NotifyBucketChangeCommand>(bucket, info); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::NotifyBucketChangeReply& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const { onEncodeReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd,BBuf& buf) const { - api::NotifyBucketChangeReply::UP msg(new api::NotifyBucketChangeReply( - static_cast<const api::NotifyBucketChangeCommand&>(cmd))); + auto msg = std::make_unique<api::NotifyBucketChangeReply>(static_cast<const api::NotifyBucketChangeCommand&>(cmd)); onDecodeReply(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::SplitBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putByte(msg.getMinSplitBits()); @@ -344,7 +320,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::SplitBucketCommand::UP msg(new api::SplitBucketCommand(bucket)); + auto msg = std::make_unique<api::SplitBucketCommand>(bucket); msg->setMinSplitBits(SH::getByte(buf)); msg->setMaxSplitBits(SH::getByte(buf)); msg->setMinByteSize(SH::getInt(buf)); @@ -353,30 +329,24 @@ ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf&, const api::SetBucketStateCommand&) const +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateCommand&) const { - throw vespalib::IllegalStateException("Unsupported serialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); } api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSetBucketStateCommand(BBuf&) const { - throw vespalib::IllegalStateException("Unsupported deserialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); } -void ProtocolSerialization4_2::onEncode( - GBBuf&, const api::SetBucketStateReply&) const +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateReply&) const { - throw vespalib::IllegalStateException("Unsupported serialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, - BBuf&) const +ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, BBuf&) const { throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); } @@ -404,11 +374,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateVisitorCommand& buf.putBoolean(msg.getFieldSet() == "[header]"); buf.putBoolean(msg.visitInconsistentBuckets()); buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - - uint32_t size = msg.getParameters().getSerializedSize(); - char* docBuffer = buf.allocate(size); - document::ByteBuffer bbuf(docBuffer, size); - msg.getParameters().serialize(bbuf); + msg.getParameters().serialize(buf); onEncodeCommand(buf, msg); } @@ -420,8 +386,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); - api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); + auto msg = std::make_unique<api::CreateVisitorCommand>(bucketSpace, libraryName, instanceId, selection); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); @@ -450,7 +415,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const msg->setVisitInconsistentBuckets(); } msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(getTypeRepo(), buf); + msg->getParameters().deserialize(buf); onDecodeCommand(buf, *msg); msg->setVisitorDispatcherVersion(42); @@ -458,8 +423,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const } void -ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::DestroyVisitorCommand& msg) const +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const { buf.putString(msg.getInstanceId()); onEncodeCommand(buf, msg); @@ -469,7 +433,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeDestroyVisitorCommand(BBuf& buf) const { vespalib::stringref instanceId = SH::getString(buf); - api::DestroyVisitorCommand::UP msg(new api::DestroyVisitorCommand(instanceId)); + auto msg = std::make_unique<api::DestroyVisitorCommand>(instanceId); onDecodeCommand(buf, *msg); return msg; } @@ -483,7 +447,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorReply& m api::StorageReply::UP ProtocolSerialization4_2::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const { - api::DestroyVisitorReply::UP msg(new api::DestroyVisitorReply(static_cast<const api::DestroyVisitorCommand&>(cmd))); + auto msg = std::make_unique<api::DestroyVisitorReply>(static_cast<const api::DestroyVisitorCommand&>(cmd)); onDecodeReply(buf, *msg); return msg; } @@ -502,8 +466,7 @@ ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const vespalib::stringref documentSelection = SH::getString(buf); document::Bucket bucket = getBucket(buf); - api::RemoveLocationCommand::UP msg; - msg.reset(new api::RemoveLocationCommand(documentSelection, bucket)); + auto msg = std::make_unique<api::RemoveLocationCommand>(documentSelection, bucket); onDecodeCommand(buf, *msg); return msg; } @@ -517,7 +480,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationReply& m api::StorageReply::UP ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - api::RemoveLocationReply::UP msg(new api::RemoveLocationReply(static_cast<const api::RemoveLocationCommand&>(cmd))); + auto msg = std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); return msg; } @@ -525,15 +488,13 @@ ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf // Utility functions for serialization void -ProtocolSerialization4_2::onEncodeBucketInfoCommand( - GBBuf& buf, const api::BucketInfoCommand& msg) const +ProtocolSerialization4_2::onEncodeBucketInfoCommand(GBBuf& buf, const api::BucketInfoCommand& msg) const { onEncodeCommand(buf, msg); } void -ProtocolSerialization4_2::onDecodeBucketInfoCommand( - BBuf& buf, api::BucketInfoCommand& msg) const +ProtocolSerialization4_2::onDecodeBucketInfoCommand(BBuf& buf, api::BucketInfoCommand& msg) const { onDecodeCommand(buf, msg); } @@ -547,8 +508,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ReturnCode& rc) const } void -ProtocolSerialization4_2::onEncodeDiffEntry( - GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const +ProtocolSerialization4_2::onEncodeDiffEntry(GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const { buf.putLong(entry._timestamp); SH::putGlobalId(entry._gid, buf); @@ -559,8 +519,7 @@ ProtocolSerialization4_2::onEncodeDiffEntry( } void -ProtocolSerialization4_2::onDecodeDiffEntry( - BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const +ProtocolSerialization4_2::onDecodeDiffEntry(BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const { entry._timestamp = SH::getLong(buf); entry._gid = SH::getGlobalId(buf); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp index b0a1685ed8c..0b1f66127ba 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp @@ -62,32 +62,26 @@ ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); api::SetBucketStateCommand::BUCKET_STATE state( - static_cast<api::SetBucketStateCommand::BUCKET_STATE>( - SH::getByte(buf))); - api::SetBucketStateCommand::UP msg( - new api::SetBucketStateCommand(bucket, state)); + static_cast<api::SetBucketStateCommand::BUCKET_STATE>(SH::getByte(buf))); + auto msg = std::make_unique<api::SetBucketStateCommand>(bucket, state); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::SetBucketStateReply& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const { onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const { - api::SetBucketStateReply::UP msg(new api::SetBucketStateReply( - static_cast<const api::SetBucketStateCommand&>(cmd))); + auto msg = std::make_unique<api::SetBucketStateReply>(static_cast<const api::SetBucketStateCommand&>(cmd)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::GetCommand& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -103,15 +97,13 @@ ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); std::string fieldSet(SH::getString(buf)); - api::GetCommand::UP msg( - new api::GetCommand(bucket, did, fieldSet, beforeTimestamp)); + auto msg = std::make_unique<api::GetCommand>(bucket, did, fieldSet, beforeTimestamp); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void -ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::CreateVisitorCommand& msg) const +ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const { putBucketSpace(msg.getBucketSpace(), buf); buf.putString(msg.getLibraryName()); @@ -133,11 +125,7 @@ ProtocolSerialization5_1::onEncode( buf.putString(msg.getFieldSet()); buf.putBoolean(msg.visitInconsistentBuckets()); buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - - uint32_t size = msg.getParameters().getSerializedSize(); - char* docBuffer = buf.allocate(size); - document::ByteBuffer bbuf(docBuffer, size); - msg.getParameters().serialize(bbuf); + msg.getParameters().serialize(buf); onEncodeCommand(buf, msg); @@ -152,8 +140,7 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); - api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); + auto msg = std::make_unique<api::CreateVisitorCommand>(bucketSpace, libraryName, instanceId, selection); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); @@ -182,17 +169,16 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const msg->setVisitInconsistentBuckets(); } msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(getTypeRepo(), buf); + msg->getParameters().deserialize(buf); onDecodeCommand(buf, *msg); SH::getInt(buf); // Unused msg->setMaxBucketsPerVisitor(SH::getInt(buf)); msg->setVisitorDispatcherVersion(50); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::CreateBucketCommand& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putBoolean(msg.getActive()); @@ -204,10 +190,10 @@ ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); bool setActive = SH::getBoolean(buf); - api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); + auto msg = std::make_unique<api::CreateBucketCommand>(bucket); msg->setActive(setActive); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index ea002ab98ed..9751fd1be98 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -85,7 +85,7 @@ std::shared_ptr<document::Document> get_document(const protobuf::Document& src_d const document::DocumentTypeRepo& type_repo) { if (!src_doc.payload().empty()) { - document::ByteBuffer doc_buf(src_doc.payload().data(), src_doc.payload().size()); + vespalib::nbostream doc_buf(src_doc.payload().data(), src_doc.payload().size()); return std::make_shared<document::Document>(type_repo, doc_buf); } return std::shared_ptr<document::Document>(); diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h index 08cec601cce..24cc3e4371d 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -87,9 +87,9 @@ public: if (size == 0) { return document::Document::UP(); } else { - document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + vespalib::nbostream stream(buf.getBufferAtPos(), size); buf.incPos(size); - return document::Document::UP(new document::Document(repo, bbuf)); + return std::make_unique<document::Document>(repo, stream); } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp index 469c6a41bc7..596af6a79d4 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp @@ -3,6 +3,8 @@ #include "storagereply.h" #include "storagecommand.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/objects/nbostream.h> + using vespalib::alloc::Alloc; using vespalib::IllegalStateException; @@ -17,8 +19,8 @@ StorageReply::StorageReply(mbus::BlobRef data, const ProtocolSerialization& seri _reply() { memcpy(_buffer.get(), data.data(), _sz); - document::ByteBuffer buf(data.data(), _sz); - buf.getIntNetwork(reinterpret_cast<int32_t&>(_mbusType)); + vespalib::nbostream nbo(data.data(), _sz); + nbo >> _mbusType; } StorageReply::StorageReply(api::StorageReply::SP reply) diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index ee8aff31914..de47a52ca9d 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -3,6 +3,7 @@ #include "bucket.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/array.hpp> #include <ostream> #include <iterator> @@ -319,7 +320,7 @@ ApplyBucketDiffCommand::Entry::print( << "), headerBlob(" << _headerBlob.size() << "), bodyBlob(" << _bodyBlob.size() << ")"; if (_headerBlob.size() > 0) { - document::ByteBuffer buf(&_headerBlob[0], + vespalib::nbostream buf(&_headerBlob[0], _headerBlob.size()); if (_repo) { document::Document doc(*_repo, buf); diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index f6185d9f8e7..45af2296e8a 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -14,12 +14,12 @@ #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/document/base/globalid.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/util/array.h> #include <vespa/storageapi/defs.h> namespace document { class DocumentTypeRepo; } -namespace storage { -namespace api { +namespace storage::api { /** * @class CreateBucketCommand @@ -254,6 +254,8 @@ public: GetBucketDiffCommand::Entry _entry; vespalib::string _docName; std::vector<char> _headerBlob; + // TODO: In theory the body blob could be removed now as all is in one blob + // That will enable simplification of code in document. std::vector<char> _bodyBlob; const document::DocumentTypeRepo *_repo; @@ -282,7 +284,7 @@ public: ApplyBucketDiffCommand(const document::Bucket &bucket, const std::vector<Node>& nodes, uint32_t maxBufferSize); - ~ApplyBucketDiffCommand(); + ~ApplyBucketDiffCommand() override; const std::vector<Node>& getNodes() const { return _nodes; } const std::vector<Entry>& getDiff() const { return _diff; } @@ -482,5 +484,4 @@ public: DECLARE_STORAGEREPLY(SetBucketStateReply, onSetBucketStateReply) }; -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp index 1463f42abeb..7fd789a8c81 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ b/storageapi/src/vespa/storageapi/message/persistence.cpp @@ -60,7 +60,7 @@ PutCommand::print(std::ostream& out, bool verbose, const std::string& indent) co { out << "Put(" << getBucketId() << ", " << _doc->getId() << ", timestamp " << _timestamp << ", size " - << _doc->serialize()->getLength() << ")"; + << _doc->serialize().size() << ")"; if (verbose) { out << " {\n" << indent << " "; _doc->print(out, verbose, indent + " "); diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index aeb58f30fb4..e531e73fea5 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -65,7 +65,7 @@ CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) { } -CreateVisitorCommand::~CreateVisitorCommand() {} +CreateVisitorCommand::~CreateVisitorCommand() = default; document::Bucket CreateVisitorCommand::getBucket() const @@ -141,8 +141,7 @@ DestroyVisitorCommand::DestroyVisitorCommand(vespalib::stringref instanceId) } void -DestroyVisitorCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const +DestroyVisitorCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DestroyVisitorCommand(" << _instanceId << ")"; if (verbose) { @@ -157,8 +156,7 @@ DestroyVisitorReply::DestroyVisitorReply(const DestroyVisitorCommand& cmd) } void -DestroyVisitorReply::print(std::ostream& out, bool verbose, - const std::string& indent) const +DestroyVisitorReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DestroyVisitorReply()"; if (verbose) { @@ -175,12 +173,10 @@ VisitorInfoCommand::VisitorInfoCommand() { } -VisitorInfoCommand::~VisitorInfoCommand() { -} +VisitorInfoCommand::~VisitorInfoCommand() = default; void -VisitorInfoCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const +VisitorInfoCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "VisitorInfoCommand("; if (_completed) { out << "completed"; } @@ -205,8 +201,7 @@ VisitorInfoReply::VisitorInfoReply(const VisitorInfoCommand& cmd) } void -VisitorInfoReply::print(std::ostream& out, bool verbose, - const std::string& indent) const +VisitorInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "VisitorInfoReply("; if (_completed) { out << "completed"; } diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp index 68fbca75393..1868e53e4a5 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp @@ -3,8 +3,7 @@ #include "returncode.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { ReturnCode::ReturnCode() : _result(OK), @@ -14,58 +13,32 @@ ReturnCode::ReturnCode() ReturnCode::ReturnCode(const ReturnCode &) = default; ReturnCode & ReturnCode::operator = (const ReturnCode &) = default; ReturnCode & ReturnCode::operator = (ReturnCode &&) = default; -ReturnCode::~ReturnCode() {} +ReturnCode::~ReturnCode() = default; ReturnCode::ReturnCode(Result result, vespalib::stringref msg) : _result(result), _message(msg) {} -ReturnCode::ReturnCode(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buffer) - : _result(OK), - _message() -{ - deserialize(repo, buffer); -} - -void ReturnCode:: -onDeserialize(const document::DocumentTypeRepo &, document::ByteBuffer& buffer) -{ - int32_t result; - buffer.getInt(result); - _result = static_cast<Result>(result); - int32_t size; - buffer.getInt(size); - const char * p = buffer.getBufferAtPos(); - buffer.incPos(size); - _message.assign(p, size); -} - -void ReturnCode::onSerialize(document::ByteBuffer& buffer) const -{ - buffer.putInt(_result); - buffer.putInt(_message.size()); - buffer.putBytes(_message.c_str(), _message.size()); -} - -size_t ReturnCode::getSerializedSize() const -{ - return 2 * sizeof(int32_t) + _message.size(); +vespalib::string ReturnCode::getResultString(Result result) { + return documentapi::DocumentProtocol::getErrorName(result); } -void -ReturnCode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "ReturnCode(" << ReturnCode::getResultString(getResult()); - if (getMessage().size() > 0) out << ", " << getMessage(); - out << ")"; +vespalib::string +ReturnCode::toString() const { + vespalib::string ret = "ReturnCode("; + ret += getResultString(_result); + if ( ! _message.empty()) { + ret += ", "; + ret += _message; + } + ret += ")"; + return ret; } -vespalib::string ReturnCode::getResultString(Result result) { - return documentapi::DocumentProtocol::getErrorName(result); +std::ostream & +operator << (std::ostream & os, const ReturnCode & returnCode) { + return os << returnCode.toString(); } bool @@ -173,5 +146,4 @@ ReturnCode::isBucketDisappearance() const } } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h index ccd95a81aa3..305a998918c 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.h +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.h @@ -10,20 +10,12 @@ #pragma once -#include <vespa/vespalib/util/printable.h> -#include <vespa/document/util/serializable.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <iosfwd> -namespace document { - class ByteBuffer; -} -namespace storage { -namespace api { +namespace storage::api { -class ReturnCode : public document::Deserializable, - public vespalib::Printable { +class ReturnCode { public: typedef documentapi::DocumentProtocol Protocol; @@ -68,31 +60,20 @@ public: private: Result _result; vespalib::string _message; - void onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) override; - void onSerialize(document::ByteBuffer& buffer) const override; - public: ReturnCode(); explicit ReturnCode(Result result, vespalib::stringref msg = ""); - ReturnCode(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buffer); ReturnCode(const ReturnCode &); ReturnCode & operator = (const ReturnCode &); ReturnCode(ReturnCode &&) = default; ReturnCode & operator = (ReturnCode &&); ~ReturnCode(); - ReturnCode* clone() const override { return new ReturnCode(*this); } - - size_t getSerializedSize() const override; - const vespalib::string& getMessage() const { return _message; } void setMessage(vespalib::stringref message) { _message = message; } Result getResult() const { return _result; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - /** * Translate from status code to human-readable string * @param result Status code returned from getResult() @@ -121,8 +102,9 @@ public: bool isShutdownRelated() const; bool isBucketDisappearance() const; bool isNonCriticalForIntegrityChecker() const; + vespalib::string toString() const; }; +std::ostream & operator << (std::ostream & os, const ReturnCode & returnCode); -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp index 86f7cb6e16d..81cdadb3623 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp @@ -4,8 +4,7 @@ #include "storagecommand.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) : StorageMessage(cmd.getType().getReplyType(), cmd.getMsgId()), @@ -19,16 +18,14 @@ StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) setTransportContext(cmd.getTransportContext()); } -StorageReply::~StorageReply() { } +StorageReply::~StorageReply() = default; void StorageReply::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; - out << "StorageReply(" << _type.getName() << ", " - << _result.toString() << ")"; + out << "StorageReply(" << _type.getName() << ", " << _result << ")"; } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h index 4219f3e28cd..1a3bbe35eb4 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.h @@ -15,8 +15,7 @@ #include "returncode.h" #include "storagemessage.h" -namespace storage { -namespace api { +namespace storage::api { class StorageCommand; @@ -28,7 +27,7 @@ protected: ReturnCode code = ReturnCode(ReturnCode::OK)); public: - ~StorageReply(); + ~StorageReply() override; DECLARE_POINTER_TYPEDEFS(StorageReply); void setResult(const ReturnCode& r) { _result = r; } @@ -37,6 +36,4 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; -} // api -} // storage - +} diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 94026ffaa19..eb1cc7f0256 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -1108,7 +1108,7 @@ SearchVisitor::generateGroupingResults() vespalib::NBOSerializer nos(os); grouping.serialize(nos); vespalib::MallocPtr blob(os.size()); - memcpy(blob, os.c_str(), os.size()); + memcpy(blob, os.data(), os.size()); searchResult.getGroupingList().add(grouping.getId(), blob); } } diff --git a/vdslib/src/tests/container/parameterstest.cpp b/vdslib/src/tests/container/parameterstest.cpp index c54d8ae66da..95a29fb97be 100644 --- a/vdslib/src/tests/container/parameterstest.cpp +++ b/vdslib/src/tests/container/parameterstest.cpp @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/vdslib/container/parameters.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/gtest/gtest.h> -using document::DocumentTypeRepo; +using vespalib::GrowableByteBuffer; +using document::ByteBuffer; using namespace vdslib; TEST(ParametersTest, test_parameters) @@ -15,11 +17,12 @@ TEST(ParametersTest, test_parameters) par.set("number", 6); par.set("int64_t", INT64_C(8589934590)); par.set("double", 0.25); - std::unique_ptr<document::ByteBuffer> buffer(par.serialize()); - buffer->flip(); - DocumentTypeRepo repo; - Parameters par2(repo, *buffer); + GrowableByteBuffer buffer; + par.serialize(buffer); + + ByteBuffer bBuf(buffer.getBuffer(), buffer.position()); + Parameters par2(bBuf); EXPECT_EQ(vespalib::stringref("overture"), par2.get("fast")); EXPECT_EQ(vespalib::stringref("yahoo"), par2.get("overture")); @@ -35,4 +38,5 @@ TEST(ParametersTest, test_parameters) EXPECT_EQ(numberDefault, par2.get("nonexistingnumber", numberDefault)); EXPECT_EQ(int64Default, par2.get("nonexistingint64_t", int64Default)); EXPECT_EQ(doubleDefault, par2.get("nonexistingdouble", doubleDefault)); + } diff --git a/vdslib/src/vespa/vdslib/container/documentsummary.cpp b/vdslib/src/vespa/vdslib/container/documentsummary.cpp index bc8a0473ab3..f948bd64687 100644 --- a/vdslib/src/vespa/vdslib/container/documentsummary.cpp +++ b/vdslib/src/vespa/vdslib/container/documentsummary.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentsummary.h" +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> #include <algorithm> namespace vdslib { @@ -21,7 +23,7 @@ DocumentSummary::DocumentSummary(document::ByteBuffer& buf) : deserialize(buf); } -DocumentSummary::~DocumentSummary() {} +DocumentSummary::~DocumentSummary() = default; void DocumentSummary::deserialize(document::ByteBuffer& buf) { @@ -47,18 +49,18 @@ void DocumentSummary::deserialize(document::ByteBuffer& buf) } } -void DocumentSummary::serialize(document::ByteBuffer& buf) const +void DocumentSummary::serialize(vespalib::GrowableByteBuffer& buf) const { - buf.putIntNetwork(0); // Just serialize dummy 4 byte field, to avoid versioning. - buf.putIntNetwork(_summary.size()); + buf.putInt(0); // Just serialize dummy 4 byte field, to avoid versioning. + buf.putInt(_summary.size()); if ( ! _summary.empty() ) { - buf.putIntNetwork(getSummarySize()); + buf.putInt(getSummarySize()); for (size_t i(0), m(_summary.size()); i < m; i++) { Summary s(_summary[i]); buf.putBytes(s.getDocId(_summaryBuffer->c_str()), s.getTotalSize()); } for (size_t i(0), m(_summary.size()); i < m; i++) { - buf.putIntNetwork(_summary[i].getSummarySize()); + buf.putInt(_summary[i].getSummarySize()); } } } diff --git a/vdslib/src/vespa/vdslib/container/documentsummary.h b/vdslib/src/vespa/vdslib/container/documentsummary.h index f04c1fa06bf..375546920ec 100644 --- a/vdslib/src/vespa/vdslib/container/documentsummary.h +++ b/vdslib/src/vespa/vdslib/container/documentsummary.h @@ -2,9 +2,10 @@ #pragma once #include <vespa/vespalib/util/memory.h> -#include <vespa/document/util/bytebuffer.h> #include <vector> +namespace document { class ByteBuffer; } +namespace vespalib { class GrowableByteBuffer; } namespace vdslib { class DocumentSummary { @@ -29,7 +30,7 @@ public: void sort(); void deserialize(document::ByteBuffer& buf); - void serialize(document::ByteBuffer& buf) const; + void serialize(vespalib::GrowableByteBuffer& buf) const; uint32_t getSerializedSize() const; private: class Summary { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 4358c42ff29..9c843df1caa 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -1,20 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "parameters.hpp" +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/xmlstream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <ostream> using namespace vdslib; Parameters::Parameters() = default; -Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) +Parameters::Parameters(document::ByteBuffer& buffer) : _parameters() { - deserialize(repo, buffer); + deserialize(buffer); } Parameters::~Parameters() = default; @@ -28,20 +30,19 @@ size_t Parameters::getSerializedSize() const return mysize; } -void Parameters::onSerialize(document::ByteBuffer& buffer) const +void Parameters::serialize(vespalib::GrowableByteBuffer& buffer) const { - buffer.putIntNetwork(_parameters.size()); + buffer.putInt(_parameters.size()); for (const auto & entry : _parameters) { - buffer.putIntNetwork(entry.first.size()); + buffer.putInt(entry.first.size()); buffer.putBytes(entry.first.c_str(), entry.first.size()); - buffer.putIntNetwork(entry.second.size()); + buffer.putInt(entry.second.size()); buffer.putBytes(entry.second.c_str(), entry.second.size()); } } -void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) +void Parameters::deserialize(document::ByteBuffer& buffer) { - (void) repo; _parameters.clear(); int32_t mysize; buffer.getIntNetwork(mysize); @@ -88,11 +89,6 @@ Parameters::operator==(const Parameters &other) const return true; } -Parameters* Parameters::clone() const -{ - return new Parameters(*this); -} - vespalib::stringref Parameters::get(vespalib::stringref id, vespalib::stringref def) const { ParametersMap::const_iterator it = _parameters.find(id); @@ -130,7 +126,7 @@ void Parameters::print(std::ostream& out, bool verbose, const std::string& inden out << ")"; } -std::string Parameters::toString() const +vespalib::string Parameters::toString() const { vespalib::string ret; for (const auto & entry : _parameters) { diff --git a/vdslib/src/vespa/vdslib/container/parameters.h b/vdslib/src/vespa/vdslib/container/parameters.h index f3ea0543546..61649b29bbe 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.h +++ b/vdslib/src/vespa/vdslib/container/parameters.h @@ -14,18 +14,15 @@ #pragma once -#include <vespa/document/util/serializable.h> #include <vespa/document/util/xmlserializable.h> #include <vespa/vespalib/stllike/hash_map.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class GrowableByteBuffer; } +namespace document { class ByteBuffer; } namespace vdslib { -class Parameters : public document::Deserializable, - public document::XmlSerializable { +class Parameters : public document::XmlSerializable { public: typedef vespalib::stringref KeyT; class Value : public vespalib::string @@ -42,27 +39,25 @@ public: private: ParametersMap _parameters; - void onSerialize(document::ByteBuffer& buffer) const override; - void onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) override; void printXml(document::XmlOutputStream& xos) const override; public: Parameters(); - Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer); - virtual ~Parameters(); + Parameters(document::ByteBuffer& buffer); + ~Parameters(); bool operator==(const Parameters &other) const; - Parameters* clone() const override; + size_t getSerializedSize() const; - size_t getSerializedSize() const override; - - bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } - unsigned int size() const { return _parameters.size(); } + bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } + unsigned int size() const { return _parameters.size(); } bool lookup(KeyT id, ValueRef & v) const; void set(KeyT id, const void * v, size_t sz) { _parameters[id] = Value(v, sz); } void print(std::ostream& out, bool verbose, const std::string& indent) const; + void serialize(vespalib::GrowableByteBuffer& buffer) const; + void deserialize(document::ByteBuffer& buffer); // Disallow ParametersMap::const_iterator begin() const { return _parameters.begin(); } @@ -92,7 +87,7 @@ public: template<typename T> T get(KeyT id, T def) const; - std::string toString() const; + vespalib::string toString() const; }; } // vdslib diff --git a/vdslib/src/vespa/vdslib/container/searchresult.cpp b/vdslib/src/vespa/vdslib/container/searchresult.cpp index 73b19a2f8a2..20cc53e2de9 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.cpp +++ b/vdslib/src/vespa/vdslib/container/searchresult.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchresult.h" +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <algorithm> namespace vdslib { @@ -25,21 +27,21 @@ void AggregatorList::deserialize(document::ByteBuffer & buf) } } -void AggregatorList::serialize(document::ByteBuffer & buf) const +void AggregatorList::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(size()); - for (const_iterator it(begin()), mt(end()); it != mt; it++) { - buf.putIntNetwork(it->first); - buf.putIntNetwork(it->second.size()); - buf.putBytes(it->second, it->second.size()); + buf.putInt(size()); + for (const auto & entry : *this) { + buf.putInt(entry.first); + buf.putInt(entry.second.size()); + buf.putBytes(entry.second, entry.second.size()); } } uint32_t AggregatorList::getSerializedSize() const { size_t sz(sizeof(uint32_t) * (1 + 2*size())); - for (const_iterator it(begin()), mt(end()); it != mt; it++) { - sz += it->second.size(); + for (const auto & entry : *this) { + sz += entry.second.size(); } return sz; } @@ -51,7 +53,7 @@ BlobContainer::BlobContainer(size_t reserve) : _offsets.push_back(0); } -BlobContainer::~BlobContainer() {} +BlobContainer::~BlobContainer() = default; size_t BlobContainer::append(const void * v, size_t sz) { @@ -84,11 +86,11 @@ void BlobContainer::deserialize(document::ByteBuffer & buf) buf.getBytes(_blob, getSize()); } -void BlobContainer::serialize(document::ByteBuffer & buf) const +void BlobContainer::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(getCount()); + buf.putInt(getCount()); for(size_t i(0), m(getCount()); i < m; i++) { - buf.putIntNetwork(getSize(i)); + buf.putInt(getSize(i)); } buf.putBytes(_blob, getSize()); } @@ -116,7 +118,7 @@ SearchResult::SearchResult(document::ByteBuffer & buf) : deserialize(buf); } -SearchResult::~SearchResult() {} +SearchResult::~SearchResult() = default; void SearchResult::deserialize(document::ByteBuffer & buf) { @@ -143,26 +145,26 @@ void SearchResult::deserialize(document::ByteBuffer & buf) _groupingList.deserialize(buf); } -void SearchResult::serialize(document::ByteBuffer & buf) const +void SearchResult::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(_totalHits); + buf.putInt(_totalHits); uint32_t hitCount = std::min(_hits.size(), _wantedHits); - buf.putIntNetwork(hitCount); + buf.putInt(hitCount); if (hitCount > 0) { uint32_t sz = getBufCount(); - buf.putIntNetwork(sz); + buf.putInt(sz); for (size_t i(0), m(hitCount); i < m; i++) { const char * s(_hits[i].getDocId(_docIdBuffer->c_str())); buf.putBytes(s, strlen(s)+1); } for (size_t i(0), m(hitCount); i < m; i++) { - buf.putDoubleNetwork(_hits[i].getRank()); + buf.putDouble(_hits[i].getRank()); } } uint32_t sortCount = std::min(_sortBlob.getCount(), _wantedHits); - buf.putIntNetwork(sortCount); + buf.putInt(sortCount); for (size_t i(0); i < sortCount; i++) { - buf.putIntNetwork(_sortBlob.getSize(_hits[i].getIndex())); + buf.putInt(_sortBlob.getSize(_hits[i].getIndex())); } for (size_t i(0); i < sortCount; i++) { size_t sz; diff --git a/vdslib/src/vespa/vdslib/container/searchresult.h b/vdslib/src/vespa/vdslib/container/searchresult.h index 081873e2989..fc893f6b5be 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.h +++ b/vdslib/src/vespa/vdslib/container/searchresult.h @@ -2,10 +2,11 @@ #pragma once #include <vespa/vespalib/util/memory.h> -#include <vespa/document/util/bytebuffer.h> #include <vector> #include <map> +namespace document { class ByteBuffer; } +namespace vespalib { class GrowableByteBuffer; } namespace vdslib { typedef std::map<size_t, vespalib::MallocPtr> IntBlobMapT; @@ -15,7 +16,7 @@ class AggregatorList : public IntBlobMapT public: void add(size_t id, const vespalib::MallocPtr & aggrBlob); void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const; }; @@ -31,7 +32,7 @@ public: size_t getSize(size_t index) const { return _offsets[index+1] - _offsets[index]; } const void * getBuf(size_t index) const { return _blob.c_str() + _offsets[index]; } void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const { return (1 + getCount()) * sizeof(uint32_t) + getSize(); } private: typedef vespalib::MallocPtr Blob; @@ -76,7 +77,7 @@ public: void sort(); void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const; private: class Hit { diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java index 1dae2b76711..8602d89c90c 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java @@ -15,6 +15,8 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Optional; +import java.util.function.Function; /** * Base class for hosted Vespa plugin mojos. @@ -98,4 +100,15 @@ public abstract class AbstractVespaMojo extends AbstractMojo { throw new IllegalArgumentException("No valid value given"); } + protected static Optional<String> optionalOf(String value) { + return Optional.ofNullable(value) + .filter(data -> ! data.isBlank()); + } + + protected static <T> Optional<T> optionalOf(String value, Function<String, T> mapper) { + return Optional.ofNullable(value) + .filter(data -> ! data.isBlank()) + .map(mapper); + } + } diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java index 80655cf1d36..6669f771a0e 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/SubmitMojo.java @@ -23,32 +23,32 @@ public class SubmitMojo extends AbstractVespaMojo { @Parameter(property = "applicationTestZip") private String applicationTestZip; - @Parameter(property = "authorEmail", required = true) + @Parameter(property = "authorEmail") private String authorEmail; - @Parameter(property = "repository", defaultValue = "unknown") + @Parameter(property = "repository") private String repository; - @Parameter(property = "branch", defaultValue = "unknown") + @Parameter(property = "branch") private String branch; - @Parameter(property = "commit", defaultValue = "unknown") + @Parameter(property = "commit") private String commit; @Parameter(property = "sourceUrl") private String sourceUrl; @Parameter(property = "projectId") - private Long projectId; + private String projectId; @Override public void doExecute() { applicationZip = firstNonBlank(applicationZip, projectPathOf("target", "application.zip")); applicationTestZip = firstNonBlank(applicationTestZip, projectPathOf("target", "application-test.zip")); - Submission submission = new Submission(repository, branch, commit, Optional.ofNullable(sourceUrl), authorEmail, - Paths.get(applicationZip), - Paths.get(applicationTestZip), - projectId == null ? OptionalLong.empty() : OptionalLong.of(projectId)); + Submission submission = new Submission(optionalOf(repository), optionalOf(branch), optionalOf(commit), + optionalOf(sourceUrl), optionalOf(authorEmail), + Paths.get(applicationZip), Paths.get(applicationTestZip), + optionalOf(projectId, Long::parseLong)); getLog().info(controller.submit(submission, id.tenant(), id.application())); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java index f09b2ba2e50..a185b03f7af 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Metric; import java.util.Map; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.1.20 */ class DummyMetric implements Metric { diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index 3109295fbdc..daaea981b5a 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -141,7 +141,7 @@ public: size_t size() const { return left(); } size_t capacity() const { return _wbuf.size(); } bool empty() const { return size() == 0; } - const char * c_str() const { return &_rbuf[0]; } + const char * data() const { return &_rbuf[0]; } const char * peek() const { return &_rbuf[_rp]; } size_t rp() const { return _rp; } nbostream & rp(size_t pos) { if (pos > _wp) fail(eof); _rp = pos; return *this; } diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index b52cace45a5..03ebc2807f9 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -59,13 +59,13 @@ public: bool resize_inplace(size_t newSize); Alloc(const Alloc &) = delete; Alloc & operator = (const Alloc &) = delete; - Alloc(Alloc && rhs) : + Alloc(Alloc && rhs) noexcept : _alloc(rhs._alloc), _allocator(rhs._allocator) { rhs.clear(); } - Alloc & operator=(Alloc && rhs) { + Alloc & operator=(Alloc && rhs) noexcept { if (this != & rhs) { if (_alloc.first != nullptr) { _allocator->free(_alloc); diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json index 25b652b7312..05fb985dbaf 100644 --- a/zkfacade/abi-spec.json +++ b/zkfacade/abi-spec.json @@ -109,4 +109,4 @@ ], "fields": [] } -} +}
\ No newline at end of file diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index d6be27a0919..7c5b1ae319a 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.function.Function; +import java.util.logging.Logger; /** * Curator interface for Vespa. @@ -47,6 +48,8 @@ import java.util.function.Function; */ public class Curator implements AutoCloseable { + private static final Logger logger = Logger.getLogger(Curator.class.getName()); + private static final int ZK_SESSION_TIMEOUT = 30000; private static final int ZK_CONNECTION_TIMEOUT = 30000; @@ -115,7 +118,7 @@ public class Curator implements AutoCloseable { if (this.curatorFramework != null) { validateConnectionSpec(connectionSpec); validateConnectionSpec(zooKeeperEnsembleConnectionSpec); - addFakeListener(); + addLoggingListener(); curatorFramework.start(); } @@ -198,10 +201,13 @@ public class Curator implements AutoCloseable { return new InterProcessMutex(curatorFramework, lockPath); } - // To avoid getting warning in log, see ticket 6389740 - private void addFakeListener() { + private void addLoggingListener() { curatorFramework.getConnectionStateListenable().addListener((curatorFramework, connectionState) -> { - // empty, not needed now + switch (connectionState) { + case SUSPENDED: logger.info("ZK connection state change: SUSPENDED"); break; + case RECONNECTED: logger.info("ZK connection state change: RECONNECTED"); break; + case LOST: logger.warning("ZK connection state change: LOST"); break; + } }); } |