diff options
138 files changed, 1530 insertions, 377 deletions
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index e88947b3fdb..aa9b196c0e4 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -260,7 +260,6 @@ "public com.yahoo.component.Version vespaVersion()", "public java.util.Optional group()", "public boolean isExclusive()", - "public java.util.Set rotations()", "public com.yahoo.config.provision.ClusterSpec with(java.util.Optional)", "public com.yahoo.config.provision.ClusterSpec exclusive(boolean)", "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index c0099878b45..f041823bf04 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -3,14 +3,9 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import java.util.Arrays; -import java.util.Collections; -import java.util.Set; -import java.util.stream.Collectors; - /** * A node's membership in a cluster. This is a value object. - * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired][/rotationId,...]" + * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired]" * * @author bratseth */ @@ -25,26 +20,23 @@ public class ClusterMembership { private ClusterMembership(String stringValue, Version vespaVersion) { String[] components = stringValue.split("/"); - if (components.length < 4 || components.length > 7) + if (components.length < 4) throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " + - "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive][/rotationId,...]'"); + "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive]'"); boolean exclusive = false; - Set<RotationName> rotations = Collections.emptySet(); if (components.length > 4) { for (int i = 4; i < components.length; i++) { String component = components[i]; switch (component) { case "exclusive": exclusive = true; break; case "retired": retired = true; break; - default: rotations = rotationsFrom(component); break; } } } this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]), - ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive, - rotations); + ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive); this.index = Integer.parseInt(components[3]); this.stringValue = toStringValue(); } @@ -62,8 +54,7 @@ public class ClusterMembership { (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") + "/" + index + ( cluster.isExclusive() ? "/exclusive" : "") + - ( retired ? "/retired" : "") + - ( !cluster.rotations().isEmpty() ? "/" + rotationsAsString(cluster.rotations()) : ""); + ( retired ? "/retired" : ""); } @@ -121,12 +112,4 @@ public class ClusterMembership { return new ClusterMembership(cluster, index, true); } - private static Set<RotationName> rotationsFrom(String s) { - return Arrays.stream(s.split(",")).map(RotationName::from).collect(Collectors.toUnmodifiableSet()); - } - - private static String rotationsAsString(Set<RotationName> rotations) { - return rotations.stream().map(RotationName::value).collect(Collectors.joining(",")); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 35ee538178a..8ed56b98705 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.collect.ImmutableSortedSet; import com.yahoo.component.Version; import java.util.Objects; @@ -23,19 +22,13 @@ public final class ClusterSpec { private final Optional<Group> groupId; private final Version vespaVersion; private boolean exclusive; - private final Set<RotationName> rotations; - private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive, - Set<RotationName> rotations) { - if (type != Type.container && !rotations.isEmpty()) { - throw new IllegalArgumentException("Rotations can only be declared for clusters of type " + Type.container); - } + private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive) { this.type = type; this.id = id; this.groupId = groupId; this.vespaVersion = vespaVersion; this.exclusive = exclusive; - this.rotations = ImmutableSortedSet.copyOf(rotations); } /** Returns the cluster type */ @@ -57,35 +50,30 @@ public final class ClusterSpec { */ public boolean isExclusive() { return exclusive; } - /** Returns the rotations of which this cluster should be a member */ - public Set<RotationName> rotations() { - return rotations; - } - public ClusterSpec with(Optional<Group> newGroup) { - return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive); } public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive); } public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, Set.of()); + return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive); } // TODO: Remove after June 2019 public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, Set<RotationName> rotations) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive); } public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, Set.of()); + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive); } // TODO: Remove after June 2019 public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, Set<RotationName> rotations) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive); } @Override diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java index 9bd0680b691..5eee55a1886 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java @@ -6,7 +6,6 @@ import com.yahoo.component.Vtag; import org.junit.Test; import java.util.Collections; -import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -37,26 +36,25 @@ public class ClusterMembershipTest { assertTrue(instance.cluster().isExclusive()); } + // TODO: Remove after June 2019. This ensures stale rotation data is handled { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/rotation1,rotation2", Vtag.currentVersion); assertFalse(instance.retired()); assertFalse(instance.cluster().isExclusive()); - assertEquals(Set.of(RotationName.from("rotation1"), RotationName.from("rotation2")), instance.cluster().rotations()); } { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/rotation1,rotation2", Vtag.currentVersion); assertFalse(instance.retired()); assertTrue(instance.cluster().isExclusive()); - assertEquals(Set.of(RotationName.from("rotation1"), RotationName.from("rotation2")), instance.cluster().rotations()); } { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired/rotation1,rotation2", Vtag.currentVersion); assertTrue(instance.retired()); assertTrue(instance.cluster().isExclusive()); - assertEquals(Set.of(RotationName.from("rotation1"), RotationName.from("rotation2")), instance.cluster().rotations()); } + // end TODO } @Test @@ -101,7 +99,6 @@ public class ClusterMembershipTest { assertFalse(instance.cluster().group().isPresent()); assertEquals(3, instance.index()); assertEquals("container/id1/3", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } private void assertContentService(ClusterMembership instance) { @@ -111,7 +108,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertFalse(instance.retired()); assertEquals("content/id1/37", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } private void assertContentServiceWithGroup(ClusterMembership instance) { @@ -121,7 +117,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertFalse(instance.retired()); assertEquals("content/id1/4/37", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } /** Serializing a spec without a group assigned works, but not deserialization */ @@ -131,7 +126,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertTrue(instance.retired()); assertEquals("content/id1/37/retired", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } private void assertContentServiceWithGroupAndRetire(ClusterMembership instance) { @@ -141,7 +135,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertTrue(instance.retired()); assertEquals("content/id1/4/37/retired", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } } diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index 7606ec58f9f..223baa6fd8d 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/md5.h> #include <vespa/vespalib/util/stringfmt.h> #include <limits> +#include <cerrno> using vespalib::string; using vespalib::stringref; diff --git a/document/src/vespa/document/datatype/referencedatatype.cpp b/document/src/vespa/document/datatype/referencedatatype.cpp index bc91f6b30ed..6caa6fdf7a9 100644 --- a/document/src/vespa/document/datatype/referencedatatype.cpp +++ b/document/src/vespa/document/datatype/referencedatatype.cpp @@ -3,6 +3,7 @@ #include "referencedatatype.h" #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/vespalib/util/exceptions.h> +#include <ostream> using vespalib::make_string; using vespalib::IllegalArgumentException; diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp index 8d928f7fb12..2f7f2208cbe 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/polymorphicarrays.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> +#include <ostream> LOG_SETUP(".document.fieldvalue.array"); diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp index 9c1b101e4ab..b5464401df2 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <cassert> #include <algorithm> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.map"); diff --git a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp index dcda102f656..281161fccbf 100644 --- a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> #include <cassert> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::make_string; diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp index d79535d54cd..1d38653dba2 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp @@ -11,6 +11,7 @@ #include <vespa/document/serialization/annotationdeserializer.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentserializer.h> +#include <ostream> using vespalib::nbostream; using vespalib::ConstBufferRef; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index ac37970213c..078e4d0ec21 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -14,6 +14,7 @@ #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <algorithm> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.structfieldvalue"); diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp index ba0338b9b61..0b55407092e 100644 --- a/document/src/vespa/document/select/doctype.cpp +++ b/document/src/vespa/document/select/doctype.cpp @@ -6,6 +6,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> +#include <ostream> namespace document::select { diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp index 85dbef5ad9b..eaa795549bf 100644 --- a/document/src/vespa/document/select/operator.cpp +++ b/document/src/vespa/document/select/operator.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <cassert> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.select.operator"); diff --git a/document/src/vespa/document/select/simpleparser.cpp b/document/src/vespa/document/select/simpleparser.cpp index a19d6086abe..879a1f195e8 100644 --- a/document/src/vespa/document/select/simpleparser.cpp +++ b/document/src/vespa/document/select/simpleparser.cpp @@ -2,6 +2,7 @@ #include "simpleparser.h" #include "compare.h" +#include <cerrno> namespace document { diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index 6b4bf15fc3b..5ebf527b82b 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -3,6 +3,7 @@ #include "value.h" #include "operator.h" #include <vespa/document/fieldvalue/fieldvalue.h> +#include <ostream> namespace document { namespace select { diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp index e5a99b49a9e..e1132b2b571 100644 --- a/document/src/vespa/document/update/addvalueupdate.cpp +++ b/document/src/vespa/document/update/addvalueupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp index 3af9350062e..90286da521a 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp +++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index bec717874dc..63e61ed3221 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/numeric/conversion/cast.hpp> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.update.fieldpathupdate"); diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp index cfedc1eb01b..48b30f13437 100644 --- a/document/src/vespa/document/update/assignvalueupdate.cpp +++ b/document/src/vespa/document/update/assignvalueupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index 48f32cb7d65..f2b8d40e0a3 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -12,6 +12,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp index d36a134ecaa..3498d14d96e 100644 --- a/document/src/vespa/document/update/fieldupdate.cpp +++ b/document/src/vespa/document/update/fieldupdate.cpp @@ -5,6 +5,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> +#include <ostream> namespace document { diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp index be970b3c30a..0837615e4fb 100644 --- a/document/src/vespa/document/update/mapvalueupdate.cpp +++ b/document/src/vespa/document/update/mapvalueupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/objects/nbostream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp index fdbee3cb394..60176d93dce 100644 --- a/document/src/vespa/document/update/removevalueupdate.cpp +++ b/document/src/vespa/document/update/removevalueupdate.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index 2c10e070bbd..fa0a99de461 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -2,6 +2,7 @@ #include "operation.h" #include <vespa/vespalib/util/approx.h> +#include <algorithm> namespace vespalib::eval::operation { diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h index 3bca340cb90..3dffe1fc089 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/fastos/src/vespa/fastos/unix_file.h @@ -10,6 +10,7 @@ #pragma once #include <vespa/fastos/file.h> +#include <cerrno> /** * This is the generic UNIX implementation of @ref FastOS_FileInterface. diff --git a/fbench/src/util/clientstatus.h b/fbench/src/util/clientstatus.h index 9b15cdf4095..f8a223e121a 100644 --- a/fbench/src/util/clientstatus.h +++ b/fbench/src/util/clientstatus.h @@ -3,6 +3,7 @@ #include <map> #include <vector> +#include <string> /** * This is a helper struct that is used by the @ref Client class to diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 5a38154b7c0..421d946c5db 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -6,7 +6,9 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -63,11 +65,21 @@ public abstract class ControllerHttpClient { } /** Creates an HTTP client against the given endpoint, which uses the given key to authenticate as the given application. */ + public static ControllerHttpClient withSignatureKey(URI endpoint, String privateKey, ApplicationId id) { + return new SigningControllerHttpClient(endpoint, privateKey, id); + } + + /** Creates an HTTP client against the given endpoint, which uses the given key to authenticate as the given application. */ public static ControllerHttpClient withSignatureKey(URI endpoint, Path privateKeyFile, ApplicationId id) { return new SigningControllerHttpClient(endpoint, privateKeyFile, id); } /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */ + public static ControllerHttpClient withKeyAndCertificate(URI endpoint, String privateKey, String certificate) { + return new MutualTlsControllerHttpClient(endpoint, privateKey, certificate); + } + + /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */ public static ControllerHttpClient withKeyAndCertificate(URI endpoint, Path privateKeyFile, Path certificateFile) { return new MutualTlsControllerHttpClient(endpoint, privateKeyFile, certificateFile); } @@ -299,9 +311,13 @@ public abstract class ControllerHttpClient { private final RequestSigner signer; - private SigningControllerHttpClient(URI endpoint, Path privateKeyFile, ApplicationId id) { + private SigningControllerHttpClient(URI endpoint, String privateKey, ApplicationId id) { super(endpoint, HttpClient.newBuilder()); - this.signer = new RequestSigner(unchecked(() -> Files.readString(privateKeyFile, UTF_8)), id.serializedForm()); + this.signer = new RequestSigner(privateKey, id.serializedForm()); + } + + private SigningControllerHttpClient(URI endpoint, Path privateKeyFile, ApplicationId id) { + this(endpoint, unchecked(() -> Files.readString(privateKeyFile, UTF_8)), id); } @Override @@ -317,7 +333,18 @@ public abstract class ControllerHttpClient { private MutualTlsControllerHttpClient(URI endpoint, Path privateKeyFile, Path certificateFile) { super(endpoint, - HttpClient.newBuilder().sslContext(new SslContextBuilder().withKeyStore(privateKeyFile, certificateFile).build())); + HttpClient.newBuilder() + .sslContext(new SslContextBuilder().withKeyStore(privateKeyFile, + certificateFile) + .build())); + } + + private MutualTlsControllerHttpClient(URI endpoint, String privateKey, String certificate) { + super(endpoint, + HttpClient.newBuilder() + .sslContext(new SslContextBuilder().withKeyStore(KeyUtils.fromPemEncodedPrivateKey(privateKey), + X509CertificateUtils.certificateListFromPem(certificate)) + .build())); } } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java index dc53439ef3b..9d85ec9bf6b 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java @@ -29,6 +29,7 @@ public class RequestVerifier { this(pemPublicKey, Clock.systemUTC()); } + /** Creates a new request verifier from the given PEM encoded ECDSA public key, with the given clock. */ public RequestVerifier(String pemPublicKey, Clock clock) { this.verifier = SignatureUtils.createVerifier(KeyUtils.fromPemEncodedPublicKey(pemPublicKey), SHA256_WITH_ECDSA); this.clock = clock; diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp index 9fb731d7583..b4238dc0a7c 100644 --- a/metrics/src/vespa/metrics/metricset.cpp +++ b/metrics/src/vespa/metrics/metricset.cpp @@ -8,6 +8,7 @@ #include <list> #include <cassert> #include <algorithm> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".metrics.metricsset"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index a6d311604fd..58c576d3f44 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -1,12 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.RotationName; import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer; import java.util.Objects; -import java.util.Set; /** * Represents a load balancer for an application's cluster. This is immutable. @@ -17,13 +14,11 @@ public class LoadBalancer { private final LoadBalancerId id; private final LoadBalancerInstance instance; - private final Set<RotationName> rotations; private final boolean inactive; - public LoadBalancer(LoadBalancerId id, LoadBalancerInstance instance, Set<RotationName> rotations, boolean inactive) { + public LoadBalancer(LoadBalancerId id, LoadBalancerInstance instance, boolean inactive) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.instance = Objects.requireNonNull(instance, "instance must be non-null"); - this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); this.inactive = inactive; } @@ -32,11 +27,6 @@ public class LoadBalancer { return id; } - /** The rotations of which this is a member */ - public Set<RotationName> rotations() { - return rotations; - } - /** The instance associated with this */ public LoadBalancerInstance instance() { return instance; @@ -52,7 +42,7 @@ public class LoadBalancer { /** Return a copy of this that is set inactive */ public LoadBalancer deactivate() { - return new LoadBalancer(id, instance, rotations, true); + return new LoadBalancer(id, instance, true); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index 17f2d7364a6..a4b915a6128 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RotationName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -35,7 +34,6 @@ public class LoadBalancerSerializer { private static final String portsField = "ports"; private static final String networksField = "networks"; private static final String realsField = "reals"; - private static final String rotationsField = "rotations"; private static final String nameField = "name"; private static final String ipAddressField = "ipAddress"; private static final String portField = "port"; @@ -58,11 +56,6 @@ public class LoadBalancerSerializer { realObject.setString(ipAddressField, real.ipAddress()); realObject.setLong(portField, real.port()); }); - Cursor rotationArray = root.setArray(rotationsField); - loadBalancer.rotations().forEach(rotation -> { - Cursor rotationObject = rotationArray.addObject(); - rotationObject.setString(nameField, rotation.value()); - }); root.setBool(inactiveField, loadBalancer.inactive()); try { @@ -89,11 +82,6 @@ public class LoadBalancerSerializer { Set<String> networks = new LinkedHashSet<>(); object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); - Set<RotationName> rotations = new LinkedHashSet<>(); - object.field(rotationsField).traverse((ArrayTraverser) (i, rotation) -> { - rotations.add(RotationName.from(rotation.field(nameField).asString())); - }); - return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), new LoadBalancerInstance( HostName.from(object.field(hostnameField).asString()), @@ -102,7 +90,6 @@ public class LoadBalancerSerializer { networks, reals ), - rotations, object.field(inactiveField).asBool()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f5f8ed53d2a..372dca84a53 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -57,7 +57,7 @@ public class LoadBalancerProvisioner { LoadBalancerInstance instance = create(application, kv.getKey().id(), kv.getValue()); // Load balancer is always re-activated here to avoid reallocation if an application/cluster is // deleted and then redeployed. - LoadBalancer loadBalancer = new LoadBalancer(id, instance, kv.getKey().rotations(), false); + LoadBalancer loadBalancer = new LoadBalancer(id, instance, false); loadBalancers.put(loadBalancer.id(), loadBalancer); db.writeLoadBalancer(loadBalancer); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index 69e13f77a09..d31834567ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -76,11 +76,7 @@ public class LoadBalancersResponse extends HttpResponse { realObject.setLong("port", real.port()); }); - Cursor rotationArray = lbObject.setArray("rotations"); - lb.rotations().forEach(rotation -> { - Cursor rotationObject = rotationArray.addObject(); - rotationObject.setString("name", rotation.value()); - }); + lbObject.setArray("rotations"); // To avoid changing the API. This can be removed when clients stop expecting this lbObject.setBool("inactive", lb.inactive()); }); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 6de93c2ae65..460764b50db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RotationName; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; @@ -39,8 +38,6 @@ public class LoadBalancerSerializerTest { new Real(HostName.from("real-2"), "127.0.0.2", 4080))), - ImmutableSet.of(RotationName.from("eu-cluster"), - RotationName.from("us-cluster")), false); LoadBalancer serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); @@ -49,7 +46,6 @@ public class LoadBalancerSerializerTest { assertEquals(loadBalancer.instance().dnsZone(), serialized.instance().dnsZone()); assertEquals(loadBalancer.instance().ports(), serialized.instance().ports()); assertEquals(loadBalancer.instance().networks(), serialized.instance().networks()); - assertEquals(loadBalancer.rotations(), serialized.rotations()); assertEquals(loadBalancer.inactive(), serialized.inactive()); assertEquals(loadBalancer.instance().reals(), serialized.instance().reals()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 58c0b3ed9cc..f97460713a5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -61,7 +61,6 @@ public class LoadBalancerProvisionerTest { assertEquals(4080, get(loadBalancers.get().get(0).instance().reals(), 0).port()); assertEquals("127.0.0.2", get(loadBalancers.get().get(0).instance().reals(), 1).ipAddress()); assertEquals(4080, get(loadBalancers.get().get(0).instance().reals(), 1).port()); - assertEquals(rotationsCluster1, loadBalancers.get().get(0).rotations()); // A container is failed Supplier<List<Node>> containers = () -> tester.getNodes(app1).type(ClusterSpec.Type.container).asList(); @@ -105,7 +104,6 @@ public class LoadBalancerProvisionerTest { .map(Real::hostname) .sorted() .collect(Collectors.toList()); - assertEquals(rotationsCluster2, loadBalancers.get().get(1).rotations()); assertEquals(activeContainers, reals); // Application is removed and load balancer is deactivated @@ -120,6 +120,7 @@ <module>standalone-container</module> <module>statistics</module> <module>storage</module> + <module>tenant-auth</module> <module>tenant-base</module> <module>tenant-cd</module> <module>testutil</module> diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 268fe63ba4c..672e7f78784 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -4,6 +4,7 @@ #include "blueprintbuilder.h" #include "termdatafromnode.h" #include "same_element_builder.h" +#include <vespa/searchcorespi/index/indexsearchable.h> #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h index 02aedf15d6e..fe9c20112f4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h @@ -52,7 +52,7 @@ public: search::queryeval::ISourceSelector &selector() { return *_selector; } // Implements ISearchContext - search::queryeval::Searchable &getIndexes() override { + IndexSearchable &getIndexes() override { return *_indexes; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h index 2965e3796bf..dc840dc79ff 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h @@ -6,6 +6,8 @@ #include <memory> +namespace searchcorespi { class IndexSearchable; } + namespace proton::matching { /** @@ -31,13 +33,14 @@ public: ISearchContext & operator = (const ISearchContext &) = delete; typedef search::queryeval::Searchable Searchable; + using IndexSearchable = searchcorespi::IndexSearchable; /** * Obtain the index fields searchable. * * @return index fields searchable. **/ - virtual Searchable &getIndexes() = 0; + virtual IndexSearchable &getIndexes() = 0; /** * Obtain the attribute fields searchable. diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 4a944dc3214..5d1e2212c83 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -9,6 +9,7 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_tools"); #include <vespa/searchlib/query/tree/querytreecreator.h> +#include <vespa/searchcorespi/index/indexsearchable.h> using search::attribute::IAttributeContext; using search::queryeval::IRequestContext; @@ -158,7 +159,7 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _hardDoom(hardDoom), _query(), _match_limiter(), - _queryEnv(indexEnv, attributeContext, rankProperties), + _queryEnv(indexEnv, attributeContext, rankProperties, searchContext.getIndexes()), _mdl(), _rankSetup(rankSetup), _featureOverrides(featureOverrides), diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp index d4320c87ab2..ec48ee7164b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "queryenvironment.h" +#include <vespa/searchlib/index/i_field_length_inspector.h> using search::attribute::IAttributeContext; using search::fef::IIndexEnvironment; @@ -11,12 +12,14 @@ namespace proton::matching { QueryEnvironment::QueryEnvironment(const IIndexEnvironment &indexEnv, const IAttributeContext &attrContext, - const Properties &properties) + const Properties &properties, + const search::index::IFieldLengthInspector &field_length_inspector) : _indexEnv(indexEnv), _attrContext(attrContext), _properties(properties), _locations(1), - _terms() + _terms(), + _field_length_inspector(field_length_inspector) { } @@ -53,6 +56,12 @@ QueryEnvironment::getAttributeContext() const return _attrContext; } +double +QueryEnvironment::get_average_field_length(const vespalib::string &field_name) const +{ + return _field_length_inspector.get_field_length_info(field_name).get_average_field_length(); +} + const search::fef::IIndexEnvironment & QueryEnvironment::getIndexEnvironment() const { diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h index d79ba1796f7..8f958870d52 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/location.h> +namespace search::index { class IFieldLengthInspector; } + namespace proton::matching { /** @@ -19,6 +21,7 @@ private: search::fef::Properties _properties; std::vector<const search::fef::Location *> _locations; std::vector<const search::fef::ITermData *> _terms; + const search::index::IFieldLengthInspector &_field_length_inspector; QueryEnvironment(const QueryEnvironment &); QueryEnvironment &operator=(const QueryEnvironment &); @@ -33,7 +36,8 @@ public: **/ QueryEnvironment(const search::fef::IIndexEnvironment &indexEnv, const search::attribute::IAttributeContext &attrContext, - const search::fef::Properties &properties); + const search::fef::Properties &properties, + const search::index::IFieldLengthInspector &field_length_inspector); /** * Used to edit the list of terms by the one setting up this query @@ -71,6 +75,8 @@ public: // inherited from search::fef::IQueryEnvironment const search::attribute::IAttributeContext & getAttributeContext() const override; + double get_average_field_length(const vespalib::string &field_name) const override; + // inherited from search::fef::IQueryEnvironment const search::fef::IIndexEnvironment & getIndexEnvironment() const override; diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp index d3a0ec4726f..16c86e8a4f5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp @@ -4,6 +4,7 @@ #include "querynodes.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> +#include <vespa/searchcorespi/index/indexsearchable.h> using search::queryeval::Blueprint; using search::queryeval::EmptyBlueprint; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index d53adf19f76..9445a0a5206 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -2,9 +2,10 @@ #include "disk_mem_usage_filter.h" #include "i_disk_mem_usage_listener.h" -#include <vespa/log/log.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".proton.server.disk_mem_usage_filter"); namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 772be9049db..06f19eb06cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -37,6 +37,7 @@ #include <vespa/searchlib/aggregation/forcelink.hpp> #include <vespa/searchlib/expression/forcelink.hpp> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.server.proton"); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp index ea09c60bd52..d9207ef70e1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp @@ -3,10 +3,11 @@ #include "searchcontext.h" using search::queryeval::Searchable; +using searchcorespi::IndexSearchable; namespace proton { -Searchable & +IndexSearchable & SearchContext::getIndexes() { return *_indexSearchable; @@ -23,7 +24,7 @@ uint32_t SearchContext::getDocIdLimit() return _docIdLimit; } -SearchContext::SearchContext(const Searchable::SP &indexSearchable, uint32_t docIdLimit) +SearchContext::SearchContext(const std::shared_ptr<IndexSearchable> &indexSearchable, uint32_t docIdLimit) : _indexSearchable(indexSearchable), _attributeBlueprintFactory(), _docIdLimit(docIdLimit) diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h index 71475c3decd..b9ea6b334b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h @@ -16,16 +16,16 @@ class SearchContext : public matching::ISearchContext { private: /// Snapshot of the indexes used. - Searchable::SP _indexSearchable; + std::shared_ptr<IndexSearchable> _indexSearchable; search::AttributeBlueprintFactory _attributeBlueprintFactory; uint32_t _docIdLimit; - Searchable &getIndexes() override; + IndexSearchable &getIndexes() override; Searchable &getAttributes() override; uint32_t getDocIdLimit() override; public: - SearchContext(const Searchable::SP &indexSearchable, uint32_t docIdLimit); + SearchContext(const std::shared_ptr<IndexSearchable> &indexSearchable, uint32_t docIdLimit); }; } // namespace proton diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 22a47952acb..e61c21bee1c 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -548,38 +548,39 @@ TEST("requireThatGrowWorks") v.setBit(103); EXPECT_EQUAL(200u, v.size()); + EXPECT_EQUAL(1023u, v.capacity()); v.invalidateCachedCount(); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); - EXPECT_TRUE(v.reserve(204)); + EXPECT_TRUE(v.reserve(1024)); EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.extend(202)); EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.shrink(200)); EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); - EXPECT_FALSE(v.reserve(204)); + EXPECT_FALSE(v.reserve(2047)); EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.shrink(202)); EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.shrink(100)); EXPECT_EQUAL(100u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71]", v)); EXPECT_EQUAL(3u, v.countTrueBits()); g.transferHoldLists(1); diff --git a/searchlib/src/tests/features/bm25/bm25_test.cpp b/searchlib/src/tests/features/bm25/bm25_test.cpp index 84bafcfa0ed..1a3895d7e28 100644 --- a/searchlib/src/tests/features/bm25/bm25_test.cpp +++ b/searchlib/src/tests/features/bm25/bm25_test.cpp @@ -11,6 +11,7 @@ using namespace search::features; using namespace search::fef; +using namespace search::fef::objectstore; using CollectionType = FieldInfo::CollectionType; using StringVector = std::vector<vespalib::string>; @@ -40,12 +41,13 @@ struct Bm25BlueprintTest : public ::testing::Test { EXPECT_FALSE(blueprint->setup(index_env, params)); } - void expect_setup_succeed(const StringVector& params) { + Blueprint::SP expect_setup_succeed(const StringVector& params) { auto blueprint = make_blueprint(); test::DummyDependencyHandler deps(*blueprint); EXPECT_TRUE(blueprint->setup(index_env, params)); EXPECT_EQ(0, deps.input.size()); EXPECT_EQ(StringVector({"score"}), deps.output); + return blueprint; } }; @@ -70,6 +72,15 @@ TEST_F(Bm25BlueprintTest, blueprint_setup_succeeds_for_index_field) expect_setup_succeed({"iws"}); } +TEST_F(Bm25BlueprintTest, blueprint_can_prepare_shared_state_with_average_field_length) +{ + auto blueprint = expect_setup_succeed({"is"}); + test::QueryEnvironment query_env; + query_env.get_avg_field_lengths()["is"] = 10; + ObjectStore store; + blueprint->prepareSharedState(query_env, store); + EXPECT_DOUBLE_EQ(10, as_value<double>(*store.get("bm25.afl.is"))); +} struct Bm25ExecutorTest : public ::testing::Test { BlueprintFactory factory; @@ -87,9 +98,10 @@ struct Bm25ExecutorTest : public ::testing::Test { test.getQueryEnv().getBuilder().addIndexNode({"foo"}); test.getQueryEnv().getBuilder().addIndexNode({"foo"}); test.getQueryEnv().getBuilder().addIndexNode({"bar"}); - + test.getQueryEnv().getBuilder().set_avg_field_length("foo", 10); + } + void setup() { EXPECT_TRUE(test.setup()); - match_data = test.createMatchDataBuilder(); clear_term(0, 0); clear_term(1, 0); @@ -111,19 +123,21 @@ struct Bm25ExecutorTest : public ::testing::Test { tfmd->setFieldLength(field_length); } - feature_t get_score(feature_t num_occs, feature_t field_length) const { - return (num_occs * 2.2) / (num_occs + (1.2 * (0.25 + 0.75 * field_length / 10.0))); + feature_t get_score(feature_t num_occs, feature_t field_length, double avg_field_length = 10) const { + return (num_occs * 2.2) / (num_occs + (1.2 * (0.25 + 0.75 * field_length / avg_field_length))); } }; TEST_F(Bm25ExecutorTest, score_is_calculated_for_a_single_term) { + setup(); prepare_term(0, 0, 3, 20); EXPECT_TRUE(execute(get_score(3.0, 20))); } TEST_F(Bm25ExecutorTest, score_is_calculated_for_multiple_terms) { + setup(); prepare_term(0, 0, 3, 20); prepare_term(1, 0, 7, 5); EXPECT_TRUE(execute(get_score(3.0, 20) + get_score(7.0, 5.0))); @@ -131,6 +145,7 @@ TEST_F(Bm25ExecutorTest, score_is_calculated_for_multiple_terms) TEST_F(Bm25ExecutorTest, term_that_does_not_match_document_is_ignored) { + setup(); prepare_term(0, 0, 3, 20); prepare_term(1, 0, 7, 5, 123); EXPECT_TRUE(execute(get_score(3.0, 20))); @@ -138,8 +153,17 @@ TEST_F(Bm25ExecutorTest, term_that_does_not_match_document_is_ignored) TEST_F(Bm25ExecutorTest, term_searching_another_field_is_ignored) { + setup(); prepare_term(2, 1, 3, 20); EXPECT_TRUE(execute(0.0)); } +TEST_F(Bm25ExecutorTest, uses_average_field_length_from_shared_state_if_found) +{ + test.getQueryEnv().getObjectStore().add("bm25.afl.foo", std::make_unique<AnyWrapper<double>>(15)); + setup(); + prepare_term(0, 0, 3, 20); + EXPECT_TRUE(execute(get_score(3.0, 20, 15))); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 3e260e8453a..75a0e4b8c71 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -25,24 +25,28 @@ SingleBoolAttribute::~SingleBoolAttribute() getGenerationHolder().clearHoldLists(); } -bool -SingleBoolAttribute::addDoc(DocId & doc) { - size_t needSize = getNumDocs() + 1; - bool incGen = false; - if (_bv.capacity() < needSize) { +void +SingleBoolAttribute::ensureRoom(DocId docIdLimit) { + if (_bv.capacity() < docIdLimit) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); - uint32_t newSize = needSize + (needSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); - incGen = _bv.reserve(newSize); + uint32_t newSize = docIdLimit + (docIdLimit * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); + bool incGen = _bv.reserve(newSize); + if (incGen) { + incGeneration(); + } } - incGen = _bv.extend(needSize) || incGen; +} + +bool +SingleBoolAttribute::addDoc(DocId & doc) { + DocId docIdLimit = getNumDocs()+1; + ensureRoom(docIdLimit); + bool incGen = _bv.extend(docIdLimit); + assert( ! incGen); incNumDocs(); doc = getNumDocs() - 1; updateUncommittedDocIdLimit(doc); - if (incGen) { - incGeneration(); - } else { - removeAllOldGenerations(); - } + removeAllOldGenerations(); return true; } @@ -85,7 +89,7 @@ SingleBoolAttribute::onCommit() { void SingleBoolAttribute::onAddDocs(DocId docIdLimit) { - _bv.reserve(docIdLimit); + ensureRoom(docIdLimit); } void diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 789948838cb..20ec0b6d077 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -101,6 +101,7 @@ protected: return false; } private: + void ensureRoom(DocId docIdLimit); int8_t getFromEnum(EnumHandle) const override { return 0; } diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index dcee48aed1a..3de0c1f1320 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -9,14 +9,22 @@ using vespalib::GenerationHeldBase; using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; -////////////////////////////////////////////////////////////////////// -// Parameterized Constructor -////////////////////////////////////////////////////////////////////// +namespace { + +size_t computeCapacity(size_t capacity, size_t allocatedBytes) { + size_t possibleCapacity = (allocatedBytes * 8) - 1; + assert(possibleCapacity >= capacity); + return possibleCapacity; +} + +} + AllocatedBitVector::AllocatedBitVector(Index numberOfElements) : BitVector(), _capacityBits(numberOfElements), _alloc(allocatePaddedAndAligned(numberOfElements)) { + _capacityBits = computeCapacity(_capacityBits, _alloc.size()); init(_alloc.get(), 0, numberOfElements); clear(); } @@ -33,6 +41,7 @@ AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBit _capacityBits(capacityBits), _alloc(allocatePaddedAndAligned(0, numberOfElements, capacityBits)) { + _capacityBits = computeCapacity(_capacityBits, _alloc.size()); init(_alloc.get(), 0, numberOfElements); clear(); if (rhsSize > 0) { @@ -58,6 +67,7 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) : _capacityBits(capacity_), _alloc(allocatePaddedAndAligned(0, rhs.size(), capacity_)) { + _capacityBits = computeCapacity(_capacityBits, _alloc.size()); memcpy(_alloc.get(), rhs.getStart(), rhs.sizeBytes()); init(_alloc.get(), 0, rhs.size()); } @@ -65,7 +75,7 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) : ////////////////////////////////////////////////////////////////////// // Destructor ////////////////////////////////////////////////////////////////////// -AllocatedBitVector::~AllocatedBitVector() { } +AllocatedBitVector::~AllocatedBitVector() = default; void AllocatedBitVector::cleanup() @@ -78,8 +88,8 @@ AllocatedBitVector::cleanup() void AllocatedBitVector::resize(Index newLength) { - _capacityBits = newLength; - _alloc = allocatePaddedAndAligned(_capacityBits); + _alloc = allocatePaddedAndAligned(newLength); + _capacityBits = computeCapacity(newLength, _alloc.size()); init(_alloc.get(), 0, newLength); clear(); } diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index 6de255c48c9..c52c52354a1 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -32,7 +32,7 @@ public: AllocatedBitVector(Index numberOfElements, Alloc buffer, size_t offset); /** - * Creates a new bitvector with room for numberOfElements bits. + * Creates a new bitvector with size of numberOfElements bits and at least a capacity of capacity. * Copies what it can from the original vector. This is used for extending vector. */ AllocatedBitVector(Index numberOfElements, Index capacity, const void * rhsBuf, size_t rhsSize); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 2c45bc8f69a..7296842f2c1 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -52,7 +52,7 @@ BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity) assert(alloc.size()/sizeof(Word) >= words); // Clear padding size_t usedBytes = numBytes(end - start); - memset(static_cast<char *>(alloc.get()) + usedBytes, 0, sz - usedBytes); + memset(static_cast<char *>(alloc.get()) + usedBytes, 0, alloc.size() - usedBytes); return alloc; } diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index ff5d878063d..e13e3b42e3d 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -9,8 +9,7 @@ namespace search { class GrowableBitVector : public AllocatedBitVector { public: - GrowableBitVector(Index newSize, Index newCapacity, - GenerationHolder &generationHolder); + GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder); /** Will return true if a a buffer is held */ bool reserve(Index newCapacity); diff --git a/searchlib/src/vespa/searchlib/common/partialbitvector.cpp b/searchlib/src/vespa/searchlib/common/partialbitvector.cpp index e57396c0dfa..e1dc144541e 100644 --- a/searchlib/src/vespa/searchlib/common/partialbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/partialbitvector.cpp @@ -29,6 +29,6 @@ PartialBitVector::PartialBitVector(const BitVector & org, Index start, Index end setBit(size()); } -PartialBitVector::~PartialBitVector() { } +PartialBitVector::~PartialBitVector() = default; } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index f17f9459ff9..46fcdafc585 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -91,7 +91,7 @@ void LogDataStore::updateSerialNum() { LockGuard guard(_updateLock); - if (getPrevActive(guard) != NULL) { + if (getPrevActive(guard) != nullptr) { if (getActive(guard).getSerialNum() < getPrevActive(guard)->getLastPersistedSerialNum()) { getActive(guard).setSerialNum(getPrevActive(guard)->getLastPersistedSerialNum()); @@ -234,7 +234,7 @@ LogDataStore::lastSyncToken() const uint64_t lastSerial(getActive(guard).getLastPersistedSerialNum()); if (lastSerial == 0) { const FileChunk * prev = getPrevActive(guard); - if (prev != NULL) { + if (prev != nullptr) { lastSerial = prev->getLastPersistedSerialNum(); } } @@ -274,7 +274,7 @@ LogDataStore::remove(uint64_t serialNum, uint32_t lid) if (lm.valid()) { _fileChunks[lm.getFileId()]->remove(lid, lm.size()); } - lm = getActive(guard).append(serialNum, lid, NULL, 0); + lm = getActive(guard).append(serialNum, lid, nullptr, 0); assert( lm.empty() ); _lidInfo[lid] = lm; } @@ -327,7 +327,7 @@ LogDataStore::getMaxCompactGain() const void LogDataStore::flush(uint64_t syncToken) { - WriteableFileChunk * active = NULL; + WriteableFileChunk * active = nullptr; std::unique_ptr<FileChunkHolder> activeHolder; assert(syncToken == _initFlushSyncToken); { @@ -604,7 +604,7 @@ LogDataStore::getDiskBloat() const /// Do not count the holes in the last file as bloat if (i != _active) { const FileChunk * chunk = _fileChunks[i.getId()].get(); - if (chunk != NULL) { + if (chunk != nullptr) { sz += chunk->getDiskBloat(); } } @@ -916,7 +916,7 @@ LogDataStore::scanDir(const vespalib::string &dir, const vespalib::string &suffi if (file.size() > suffix.size() && file.find(suffix.c_str()) == file.size() - suffix.size()) { vespalib::string base(file.substr(0, file.find(suffix.c_str()))); - char *err(NULL); + char *err(nullptr); errno = 0; NameId baseId(strtoul(base.c_str(), &err, 10)); if ((errno == 0) && (err[0] == '\0')) { diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index c4d1e8bbdb4..4ab747d115d 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -89,7 +89,7 @@ public: const search::common::FileHeaderContext &fileHeaderContext, transactionlog::SyncProxy &tlSyncer, const IBucketizer::SP & bucketizer, bool readOnly = false); - ~LogDataStore(); + ~LogDataStore() override; // Implements IDataStore API ssize_t read(uint32_t lid, vespalib::DataBuffer & buffer) const override; @@ -220,7 +220,7 @@ private: const FileChunk * getPrevActive(const LockGuard & guard) const { assert(guard.locks(_updateLock)); (void) guard; - return ( !_prevActive.isActive() ) ? _fileChunks[_prevActive.getId()].get() : NULL; + return ( !_prevActive.isActive() ) ? _fileChunks[_prevActive.getId()].get() : nullptr; } void setActive(const LockGuard & guard, FileId fileId) { assert(guard.locks(_updateLock)); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 91f5c37b817..50517cf09e2 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -3,7 +3,7 @@ #include "writeablefilechunk.h" #include "data_store_file_chunk_stats.h" #include "summaryexceptions.h" -#include <vespa/vespalib/util/closuretask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> @@ -14,8 +14,7 @@ #include <vespa/log/log.h> LOG_SETUP(".search.writeablefilechunk"); -using vespalib::makeTask; -using vespalib::makeClosure; +using vespalib::makeLambdaTask; using vespalib::FileHeader; using vespalib::make_string; using vespalib::LockGuard; @@ -45,7 +44,6 @@ class PendingChunk uint64_t _dataOffset; uint32_t _dataLen; public: - typedef std::shared_ptr<PendingChunk> SP; PendingChunk(uint64_t lastSerial, uint64_t dataOffset, uint32_t dataLen); ~PendingChunk(); vespalib::nbostream & getSerializedIdx() { return _idx; } @@ -59,7 +57,6 @@ public: class ProcessedChunk { public: - typedef std::unique_ptr<ProcessedChunk> UP; ProcessedChunk(uint32_t chunkId, uint32_t alignment) : _chunkId(chunkId), _payLoad(0), @@ -77,7 +74,7 @@ private: }; WriteableFileChunk:: -WriteableFileChunk(vespalib::ThreadExecutor &executor, +WriteableFileChunk(vespalib::Executor &executor, FileId fileId, NameId nameId, const vespalib::string &baseName, SerialNum initialSerialNum, @@ -155,6 +152,7 @@ WriteableFileChunk::openIdx() { } return file; } + WriteableFileChunk::~WriteableFileChunk() { if (!frozen()) { @@ -177,7 +175,7 @@ WriteableFileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t s { size_t sz = FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit); _nextChunkId = _chunkInfo.size(); - _active.reset( new Chunk(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes()))); + _active = std::make_unique<Chunk>(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes())); _serialNum = getLastPersistedSerialNum(); _firstChunkIdToBeWritten = _active->getId(); setDiskFootprint(0); @@ -188,7 +186,7 @@ WriteableFileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t s void WriteableFileChunk::restart(uint32_t nextChunkId) { - _executor.execute(makeTask(makeClosure(this, &WriteableFileChunk::fileWriter, nextChunkId))); + _executor.execute(makeLambdaTask([this, nextChunkId] {fileWriter(nextChunkId);})); } namespace { @@ -219,7 +217,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB const LidInfoWithLid & li = *(begin + i); uint32_t chunk = li.getChunkId(); if ((chunk >= _chunkInfo.size()) || !_chunkInfo[chunk].valid()) { - ChunkMap::const_iterator found = _chunkMap.find(chunk); + auto found = _chunkMap.find(chunk); vespalib::ConstBufferRef buffer; if (found != _chunkMap.end()) { buffer = found->second->getLid(li.getLid()); @@ -234,8 +232,8 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB } } for (auto & it : chunksOnFile) { - LidInfoWithLidV::const_iterator first = find_first(begin, it.first); - LidInfoWithLidV::const_iterator last = seek_past(first, begin + count, it.first); + auto first = find_first(begin, it.first); + auto last = seek_past(first, begin + count, it.first); FileChunk::read(first, last - first, it.second, visitor); } } else { @@ -250,7 +248,7 @@ WriteableFileChunk::read(uint32_t lid, SubChunkId chunkId, vespalib::DataBuffer if (!frozen()) { LockGuard guard(_lock); if ((chunkId >= _chunkInfo.size()) || !_chunkInfo[chunkId].valid()) { - ChunkMap::const_iterator found = _chunkMap.find(chunkId); + auto found = _chunkMap.find(chunkId); if (found != _chunkMap.end()) { return found->second->read(lid, buffer); } else { @@ -268,13 +266,13 @@ WriteableFileChunk::read(uint32_t lid, SubChunkId chunkId, vespalib::DataBuffer void WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) { - Chunk * active(NULL); + Chunk * active(nullptr); { LockGuard guard(_lock); active = _chunkMap[chunkId].get(); } - ProcessedChunk::UP tmp(new ProcessedChunk(chunkId, _alignment)); + auto tmp = std::make_unique<ProcessedChunk>(chunkId, _alignment); if (_alignment > 1) { tmp->getBuf().ensureFree(active->getMaxPackSize(_config.getCompression()) + _alignment - 1); } @@ -293,12 +291,12 @@ WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) } void -WriteableFileChunk::enque(ProcessedChunk::UP tmp) +WriteableFileChunk::enque(ProcessedChunkUP tmp) { LOG(debug, "enqueing %p", tmp.get()); MonitorGuard guard(_writeMonitor); _writeQ.push_back(std::move(tmp)); - if (_writeTaskIsRunning == false) { + if ( ! _writeTaskIsRunning) { _writeTaskIsRunning = true; uint32_t nextChunkId = _firstChunkIdToBeWritten; guard.signal(); @@ -359,12 +357,12 @@ WriteableFileChunk::insertChunks(ProcessedChunkMap & orderedChunks, ProcessedChu { (void) nextChunkId; for (auto &chunk : newChunks) { - if (chunk.get() != 0) { + if (chunk) { assert(chunk->getChunkId() >= nextChunkId); assert(orderedChunks.find(chunk->getChunkId()) == orderedChunks.end()); orderedChunks[chunk->getChunkId()] = std::move(chunk); } else { - orderedChunks[std::numeric_limits<uint32_t>::max()] = ProcessedChunk::UP(); + orderedChunks[std::numeric_limits<uint32_t>::max()] = ProcessedChunkUP(); } } } @@ -375,7 +373,7 @@ WriteableFileChunk::fetchNextChain(ProcessedChunkMap & orderedChunks, const uint ProcessedChunkQ chunks; while (!orderedChunks.empty() && ((orderedChunks.begin()->first == (firstChunkId+chunks.size())) || - (orderedChunks.begin()->second.get() == NULL))) + !orderedChunks.begin()->second)) { chunks.push_back(std::move(orderedChunks.begin()->second)); orderedChunks.erase(orderedChunks.begin()); @@ -393,8 +391,7 @@ WriteableFileChunk::computeChunkMeta(const LockGuard & guard, const ChunkMeta cmeta(offset, tmp.getPayLoad(), active.getLastSerial(), active.count()); assert((size_t(tmp.getBuf().getData())%_alignment) == 0); assert((dataLen%_alignment) == 0); - PendingChunk::SP pcsp; - pcsp.reset(new PendingChunk(active.getLastSerial(), offset, dataLen)); + auto pcsp = std::make_shared<PendingChunk>(active.getLastSerial(), offset, dataLen); PendingChunk &pc(*pcsp.get()); nbostream &os(pc.getSerializedIdx()); cmeta.serialize(os); @@ -424,8 +421,7 @@ WriteableFileChunk::computeChunkMeta(ProcessedChunkQ & chunks, size_t startPos, LockGuard guard(_lock); if (!_pendingChunks.empty()) { - const PendingChunk::SP pcsp(_pendingChunks.back()); - const PendingChunk &pc(*pcsp.get()); + const PendingChunk & pc = *_pendingChunks.back(); assert(pc.getLastSerial() >= lastSerial); lastSerial = pc.getLastSerial(); } @@ -454,7 +450,7 @@ WriteableFileChunk::writeData(const ProcessedChunkQ & chunks, size_t sz) { vespalib::DataBuffer buf(0ul, _alignment); buf.ensureFree(sz); - for (const ProcessedChunk::UP & chunk : chunks) { + for (const auto & chunk : chunks) { buf.writeBytes(chunk->getBuf().getData(), chunk->getBuf().getDataLen()); } @@ -540,15 +536,15 @@ WriteableFileChunk::freeze() { if (!frozen()) { waitForAllChunksFlushedToDisk(); - enque(ProcessedChunk::UP()); - _executor.sync(); + enque(ProcessedChunkUP()); { MonitorGuard guard(_writeMonitor); while (_writeTaskIsRunning) { guard.wait(10); } - assert(_writeQ.empty()); } + assert(_writeQ.empty()); + assert(_chunkMap.empty()); { MonitorGuard guard(_lock); setDiskFootprint(getDiskFootprint(guard)); @@ -632,7 +628,7 @@ int32_t WriteableFileChunk::flushLastIfNonEmpty(bool force) chunkId = _active->getId(); _chunkMap[chunkId] = std::move(_active); assert(_nextChunkId < LidInfo::getChunkIdLimit()); - _active.reset(new Chunk(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes()))); + _active = std::make_unique<Chunk>(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes())); } return chunkId; } @@ -643,10 +639,7 @@ WriteableFileChunk::flush(bool block, uint64_t syncToken) int32_t chunkId = flushLastIfNonEmpty(syncToken > _serialNum); if (chunkId >= 0) { setSerialNum(syncToken); - _executor.execute(makeTask(makeClosure(this, - &WriteableFileChunk::internalFlush, - static_cast<uint32_t>(chunkId), - _serialNum))); + _executor.execute(makeLambdaTask([this, chunkId, serialNum=_serialNum] { internalFlush(chunkId, serialNum); })); } else { if (block) { MonitorGuard guard(_lock); @@ -656,7 +649,6 @@ WriteableFileChunk::flush(bool block, uint64_t syncToken) } } if (block) { - _executor.sync(); waitForChunkFlushedToDisk(chunkId); } } @@ -693,10 +685,7 @@ WriteableFileChunk::waitForAllChunksFlushedToDisk() const } LidInfo -WriteableFileChunk::append(uint64_t serialNum, - uint32_t lid, - const void * buffer, - size_t len) +WriteableFileChunk::append(uint64_t serialNum, uint32_t lid, const void * buffer, size_t len) { assert( !frozen() ); if ( ! _active->hasRoom(len)) { @@ -818,8 +807,7 @@ WriteableFileChunk::needFlushPendingChunks(const MonitorGuard & guard, uint64_t assert(guard.monitors(_lock)); if (_pendingChunks.empty()) return false; - const PendingChunk::SP pcsp(_pendingChunks.front()); - const PendingChunk &pc(*pcsp.get()); + const PendingChunk & pc = *_pendingChunks.front(); if (pc.getLastSerial() > serialNum) return false; bool datWritten = datFileLen >= pc.getDataOffset() + pc.getDataLen(); @@ -868,8 +856,7 @@ WriteableFileChunk::unconditionallyFlushPendingChunks(const vespalib::LockGuard for (;;) { if (!needFlushPendingChunks(guard, serialNum, datFileLen)) break; - PendingChunk::SP pcsp; - pcsp.swap(_pendingChunks.front()); + std::shared_ptr<PendingChunk> pcsp = std::move(_pendingChunks.front()); _pendingChunks.pop_front(); const PendingChunk &pc(*pcsp.get()); assert(_pendingIdx >= pc.getIdxLen()); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 4a2ebfc42df..2c300bc9035 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -3,7 +3,7 @@ #pragma once #include "filechunk.h" -#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/executor.h> #include <vespa/searchlib/transactionlog/syncproxy.h> #include <vespa/fastos/file.h> #include <map> @@ -42,7 +42,7 @@ public: public: typedef std::unique_ptr<WriteableFileChunk> UP; - WriteableFileChunk(vespalib::ThreadExecutor & executor, FileId fileId, NameId nameId, + WriteableFileChunk(vespalib::Executor & executor, FileId fileId, NameId nameId, const vespalib::string & baseName, uint64_t initialSerialNum, uint32_t docIdLimit, const Config & config, const TuneFileSummary &tune, const common::FileHeaderContext &fileHeaderContext, @@ -128,7 +128,7 @@ private: bool _writeTaskIsRunning; vespalib::Monitor _writeMonitor; ProcessedChunkQ _writeQ; - vespalib::ThreadExecutor & _executor; + vespalib::Executor & _executor; ProcessedChunkMap _orderedChunks; BucketDensityComputer _bucketMap; }; diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index a9430db09c3..0be3c2876f7 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -3,26 +3,29 @@ #include "bm25_feature.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/itermfielddata.h> +#include <vespa/searchlib/fef/objectstore.h> #include <memory> namespace search::features { +using fef::AnyWrapper; using fef::Blueprint; using fef::FeatureExecutor; using fef::FieldInfo; using fef::ITermData; using fef::ITermFieldData; using fef::MatchDataDetails; +using fef::objectstore::as_value; Bm25Executor::Bm25Executor(const fef::FieldInfo& field, - const fef::IQueryEnvironment& env) + const fef::IQueryEnvironment& env, + double avg_field_length) : FeatureExecutor(), _terms(), - _avg_field_length(10), + _avg_field_length(avg_field_length), _k1_param(1.2), _b_param(0.75) { - // TODO: Don't use hard coded avg_field_length // TODO: Add support for setting k1 and b for (size_t i = 0; i < env.getNumTerms(); ++i) { const ITermData* term = env.getTerm(i); @@ -93,10 +96,33 @@ Bm25Blueprint::setup(const fef::IIndexEnvironment& env, const fef::ParameterList return (_field != nullptr); } +namespace { + +vespalib::string +make_avg_field_length_key(const vespalib::string& base_name, const vespalib::string& field_name) +{ + return base_name + ".afl." + field_name; +} + +} + +void +Bm25Blueprint::prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const +{ + vespalib::string key = make_avg_field_length_key(getBaseName(), _field->name()); + if (store.get(key) == nullptr) { + store.add(key, std::make_unique<AnyWrapper<double>>(env.get_average_field_length(_field->name()))); + } +} + fef::FeatureExecutor& Bm25Blueprint::createExecutor(const fef::IQueryEnvironment& env, vespalib::Stash& stash) const { - return stash.create<Bm25Executor>(*_field, env); + const auto* lookup_result = env.getObjectStore().get(make_avg_field_length_key(getBaseName(), _field->name())); + double avg_field_length = lookup_result != nullptr ? + as_value<double>(*lookup_result) : + env.get_average_field_length(_field->name()); + return stash.create<Bm25Executor>(*_field, env, avg_field_length); } } diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.h b/searchlib/src/vespa/searchlib/features/bm25_feature.h index 457cfea4c87..4b1576b57b3 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.h +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.h @@ -30,7 +30,8 @@ private: public: Bm25Executor(const fef::FieldInfo& field, - const fef::IQueryEnvironment& env); + const fef::IQueryEnvironment& env, + double avg_field_length); void handle_bind_match_data(const fef::MatchData& match_data) override; void execute(uint32_t docId) override; @@ -53,6 +54,7 @@ public: return fef::ParameterDescriptions().desc().indexField(fef::ParameterCollection::ANY); } bool setup(const fef::IIndexEnvironment& env, const fef::ParameterList& params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; fef::FeatureExecutor& createExecutor(const fef::IQueryEnvironment& env, vespalib::Stash& stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index b9041901ced..b927188c1aa 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -15,6 +15,7 @@ #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/locale/c.h> +#include <cerrno> #include <vespa/log/log.h> LOG_SETUP(".features.queryfeature"); diff --git a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h index a7f268e5c6b..041e9ec67bc 100644 --- a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h @@ -70,6 +70,15 @@ public: virtual const search::attribute::IAttributeContext & getAttributeContext() const = 0; /** + * Returns the average field length for the given field. + * + * @param field_name field name + * + * @return average field length + **/ + virtual double get_average_field_length(const vespalib::string &field_name) const = 0; + + /** * Returns a const view of the index environment. * * @return index environment diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h index 49176afa3c9..2debcd277e9 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.h +++ b/searchlib/src/vespa/searchlib/fef/objectstore.h @@ -2,9 +2,13 @@ #pragma once #include <vespa/vespalib/stllike/hash_map.h> +#include <cassert> namespace search::fef { +/** + * Top level interface for things to store in an IObjectStore. + */ class Anything { public: @@ -12,6 +16,9 @@ public: virtual ~Anything() { } }; +/** + * Implementation of the Anything interface that wraps a value of the given type. + */ template<typename T> class AnyWrapper : public Anything { @@ -22,6 +29,9 @@ private: T _value; }; +/** + * Interface for a key value store of Anything instances. + */ class IObjectStore { public: @@ -30,6 +40,9 @@ public: virtual const Anything * get(const vespalib::string & key) const = 0; }; +/** + * Object store implementation on top of a hash map. + */ class ObjectStore : public IObjectStore { public: @@ -42,4 +55,20 @@ private: ObjectMap _objectMap; }; +namespace objectstore { + +/** + * Utility function that gets the value stored in an Anything instance (via AnyWrapper). + */ +template<typename T> +const T & +as_value(const Anything &val) { + using WrapperType = AnyWrapper<T>; + const auto *wrapper = dynamic_cast<const WrapperType *>(&val); + assert(wrapper != nullptr); + return wrapper->getValue(); +} + +} + } diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h index 4c7ca4b67d7..4e46c9eaa7c 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h @@ -113,6 +113,7 @@ public: const Properties & getProperties() const override { return _queryEnv.getProperties(); } const Location & getLocation() const override { return _queryEnv.getLocation(); } const attribute::IAttributeContext & getAttributeContext() const override { return _queryEnv.getAttributeContext(); } + double get_average_field_length(const vespalib::string &field_name) const override { return _queryEnv.get_average_field_length(field_name); } const IIndexEnvironment & getIndexEnvironment() const override { return _queryEnv.getIndexEnvironment(); } void bind_match_data(const fef::MatchData &md) { _matchData = &md; } }; diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp index ee305dcff55..4697675c071 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp @@ -2,21 +2,17 @@ #include "queryenvironment.h" -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { QueryEnvironment::QueryEnvironment(IndexEnvironment *env) : _indexEnv(env), _terms(), _properties(), _location(), - _attrCtx((env == NULL) ? attribute::IAttributeContext::UP() : env->getAttributeMap().createContext()) + _attrCtx((env == nullptr) ? attribute::IAttributeContext::UP() : env->getAttributeMap().createContext()) { } QueryEnvironment::~QueryEnvironment() { } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h index 0179b5020e6..40898281794 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h @@ -6,10 +6,9 @@ #include <vespa/searchlib/fef/iqueryenvironment.h> #include <vespa/searchlib/fef/location.h> #include <vespa/searchlib/fef/simpletermdata.h> +#include <unordered_map> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { /** * Implementation of the IQueryEnvironment interface used for testing. @@ -25,6 +24,7 @@ private: Properties _properties; Location _location; search::attribute::IAttributeContext::UP _attrCtx; + std::unordered_map<std::string, double> _avg_field_lengths; public: /** @@ -40,6 +40,13 @@ public: const ITermData *getTerm(uint32_t idx) const override { return idx < _terms.size() ? &_terms[idx] : NULL; } const Location & getLocation() const override { return _location; } const search::attribute::IAttributeContext &getAttributeContext() const override { return *_attrCtx; } + double get_average_field_length(const vespalib::string& field_name) const override { + auto itr = _avg_field_lengths.find(field_name); + if (itr != _avg_field_lengths.end()) { + return itr->second; + } + return 1.0; + } const IIndexEnvironment &getIndexEnvironment() const override { assert(_indexEnv != NULL); return *_indexEnv; } /** Returns a reference to the index environment of this. */ @@ -76,9 +83,9 @@ public: /** Returns a reference to the location of this. */ Location & getLocation() { return _location; } + + std::unordered_map<std::string, double>& get_avg_field_lengths() { return _avg_field_lengths; } }; -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp index 2d9fb998869..67a2eaf5677 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp @@ -2,16 +2,13 @@ #include "queryenvironmentbuilder.h" -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { QueryEnvironmentBuilder::QueryEnvironmentBuilder(QueryEnvironment &env, MatchDataLayout &layout) : _queryEnv(env), _layout(layout) { - // empty } QueryEnvironmentBuilder::~QueryEnvironmentBuilder() { } @@ -39,8 +36,8 @@ QueryEnvironmentBuilder::addIndexNode(const std::vector<vespalib::string> &field td.setWeight(search::query::Weight(100)); for (uint32_t i = 0; i < fieldNames.size(); ++i) { const FieldInfo *info = _queryEnv.getIndexEnv()->getFieldByName(fieldNames[i]); - if (info == NULL || info->type() != FieldType::INDEX) { - return NULL; + if (info == nullptr || info->type() != FieldType::INDEX) { + return nullptr; } SimpleTermFieldData &tfd = td.addField(info->id()); tfd.setHandle(_layout.allocTermField(tfd.getFieldId())); @@ -52,8 +49,8 @@ SimpleTermData * QueryEnvironmentBuilder::addAttributeNode(const vespalib::string &attrName) { const FieldInfo *info = _queryEnv.getIndexEnv()->getFieldByName(attrName); - if (info == NULL || info->type() != FieldType::ATTRIBUTE) { - return NULL; + if (info == nullptr || info->type() != FieldType::ATTRIBUTE) { + return nullptr; } _queryEnv.getTerms().push_back(SimpleTermData()); SimpleTermData &td = _queryEnv.getTerms().back(); @@ -63,6 +60,11 @@ QueryEnvironmentBuilder::addAttributeNode(const vespalib::string &attrName) return &td; } -} // namespace test -} // namespace fef -} // namespace search +QueryEnvironmentBuilder& +QueryEnvironmentBuilder::set_avg_field_length(const vespalib::string& field_name, double avg_field_length) +{ + _queryEnv.get_avg_field_lengths()[field_name] = avg_field_length; + return *this; +} + +} diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h index 98aed323f9a..36a63b2a9a2 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h @@ -57,6 +57,8 @@ public: /** Returns a const reference to the match data layout of this. */ const MatchDataLayout &getLayout() const { return _layout; } + QueryEnvironmentBuilder& set_avg_field_length(const vespalib::string& field_name, double avg_field_length); + private: QueryEnvironmentBuilder(const QueryEnvironmentBuilder &); // hide QueryEnvironmentBuilder & operator=(const QueryEnvironmentBuilder &); // hide diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index 638f22fc8b7..496a19d153f 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "url.h" +#include <algorithm> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.url"); diff --git a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp b/staging_vespalib/src/vespa/vespalib/util/rusage.cpp index 645be2937d6..62d0158f784 100644 --- a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/rusage.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rusage.h" #include <stdexcept> +#include <cerrno> #include <vespa/vespalib/util/stringfmt.h> namespace vespalib { diff --git a/storage/src/tests/distributor/messagesenderstub.h b/storage/src/tests/distributor/messagesenderstub.h index b86863890a1..1b526813ef7 100644 --- a/storage/src/tests/distributor/messagesenderstub.h +++ b/storage/src/tests/distributor/messagesenderstub.h @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/distributormessagesender.h> #include <cassert> #include <vector> +#include <string> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index 440a76b92fd..3cef17c9025 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <thread> #include <chrono> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/stdmapwrapper.h b/storage/src/vespa/storage/bucketdb/stdmapwrapper.h index 8cd2d6a7578..889227f1747 100644 --- a/storage/src/vespa/storage/bucketdb/stdmapwrapper.h +++ b/storage/src/vespa/storage/bucketdb/stdmapwrapper.h @@ -12,6 +12,7 @@ #include <map> #include <vespa/vespalib/util/printable.h> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index 799b3641c64..e143f4d8570 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simplemaintenancescanner.h" #include <vespa/storage/distributor/distributor_bucket_space.h> +#include <ostream> namespace storage::distributor { diff --git a/storage/src/vespa/storage/distributor/messagetracker.h b/storage/src/vespa/storage/distributor/messagetracker.h index 626335e1ba6..75ae287d98f 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.h +++ b/storage/src/vespa/storage/distributor/messagetracker.h @@ -4,6 +4,7 @@ #include <vespa/storage/common/messagesender.h> #include <vector> #include <map> +#include <string> namespace storage::api { class BucketCommand; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 784ea7253b6..66ce4fc0485 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -3,6 +3,7 @@ #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/pendingmessagetracker.h> +#include <array> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.merge"); diff --git a/storage/src/vespa/storage/persistence/diskthread.h b/storage/src/vespa/storage/persistence/diskthread.h index 0489ad3144a..926b766aca0 100644 --- a/storage/src/vespa/storage/persistence/diskthread.h +++ b/storage/src/vespa/storage/persistence/diskthread.h @@ -16,6 +16,7 @@ #include <vespa/vespalib/util/document_runnable.h> #include <vespa/config-stor-filestor.h> #include <vespa/storageframework/generic/thread/runnable.h> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp b/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp index 86c70a4a287..24f4c9cd731 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mergestatus.h" +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".mergestatus"); diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index f38d3d0fac3..61a10f71868 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages.h" +#include <ostream> using document::BucketSpace; diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp index 73f11b39e67..98c5e56b90c 100644 --- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp +++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp @@ -9,6 +9,7 @@ #include <iomanip> #include <iostream> #include <algorithm> +#include <sstream> #include <vespa/config-stor-distribution.h> namespace storage { diff --git a/storage/src/vespa/storage/visiting/commandqueue.h b/storage/src/vespa/storage/visiting/commandqueue.h index 8cc565884ec..d129506eb64 100644 --- a/storage/src/vespa/storage/visiting/commandqueue.h +++ b/storage/src/vespa/storage/visiting/commandqueue.h @@ -19,6 +19,7 @@ #include <vespa/fastos/timestamp.h> #include <vespa/storageframework/generic/clock/clock.h> #include <list> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index c8d34139364..f53ca5a60a0 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -24,6 +24,7 @@ #include <vespa/persistence/spi/read_consistency.h> #include <list> #include <deque> +#include <ostream> namespace document { class Document; diff --git a/storageapi/src/vespa/storageapi/message/datagram.cpp b/storageapi/src/vespa/storageapi/message/datagram.cpp index 66b753185a4..3376761ee41 100644 --- a/storageapi/src/vespa/storageapi/message/datagram.cpp +++ b/storageapi/src/vespa/storageapi/message/datagram.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datagram.h" +#include <ostream> using document::BucketSpace; diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.cpp b/storageapi/src/vespa/storageapi/message/documentsummary.cpp index e1bf84a8e7c..1d81c6a4c16 100644 --- a/storageapi/src/vespa/storageapi/message/documentsummary.cpp +++ b/storageapi/src/vespa/storageapi/message/documentsummary.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentsummary.h" +#include <ostream> namespace storage { namespace api { diff --git a/storageapi/src/vespa/storageapi/message/queryresult.cpp b/storageapi/src/vespa/storageapi/message/queryresult.cpp index 67b39d19f6d..bf81083b954 100644 --- a/storageapi/src/vespa/storageapi/message/queryresult.cpp +++ b/storageapi/src/vespa/storageapi/message/queryresult.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "queryresult.h" +#include <ostream> namespace storage { namespace api { diff --git a/storageapi/src/vespa/storageapi/message/searchresult.cpp b/storageapi/src/vespa/storageapi/message/searchresult.cpp index caa698fa67b..4299109e6b5 100644 --- a/storageapi/src/vespa/storageapi/message/searchresult.cpp +++ b/storageapi/src/vespa/storageapi/message/searchresult.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchresult.h" +#include <ostream> using vdslib::SearchResult; diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index f5850de98ad..8adec61a34e 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -3,6 +3,7 @@ #include "visitor.h" #include <vespa/vespalib/util/array.hpp> #include <climits> +#include <ostream> namespace storage::api { diff --git a/storageframework/src/vespa/storageframework/generic/clock/time.cpp b/storageframework/src/vespa/storageframework/generic/clock/time.cpp index 0e8178b03bb..7ead5e50077 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/time.cpp +++ b/storageframework/src/vespa/storageframework/generic/clock/time.cpp @@ -5,6 +5,7 @@ #include <iomanip> #include <vector> #include <cassert> +#include <sstream> namespace storage { namespace framework { diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h index b9391ac838c..db0f95cb6bb 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h @@ -54,6 +54,8 @@ public: // inherit documentation virtual const search::attribute::IAttributeContext & getAttributeContext() const override { return *_attrCtx; } + double get_average_field_length(const vespalib::string &) const override { return 1.0; } + // inherit documentation virtual const search::fef::IIndexEnvironment & getIndexEnvironment() const override { return _indexEnv; } diff --git a/tenant-auth/OWNERS b/tenant-auth/OWNERS new file mode 100644 index 00000000000..d0a102ecbf4 --- /dev/null +++ b/tenant-auth/OWNERS @@ -0,0 +1 @@ +jonmv diff --git a/tenant-auth/README.md b/tenant-auth/README.md new file mode 100644 index 00000000000..0514b68400e --- /dev/null +++ b/tenant-auth/README.md @@ -0,0 +1 @@ +# Utilities that authenticate users to the hosted Vespa API, or to hosted Vespa applications. diff --git a/tenant-auth/pom.xml b/tenant-auth/pom.xml new file mode 100644 index 00000000000..be8b42dd6c2 --- /dev/null +++ b/tenant-auth/pom.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>tenant-auth</artifactId> + <description>Provides resources for authenticating with the hosted Vespa API or application containers</description> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/Authenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/Authenticator.java new file mode 100644 index 00000000000..6ecf1100630 --- /dev/null +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/Authenticator.java @@ -0,0 +1,73 @@ +package ai.vespa.hosted.auth; + +import ai.vespa.hosted.api.ControllerHttpClient; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateUtils; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpRequest; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.Optional; + +/** + * Authenticates {@link HttpRequest}s against a hosted Vespa application based on mutual TLS. + * + * @author jonmv + */ +public class Authenticator { + + /** Returns an SSLContext from "key" and "cert" files found under {@code System.getProperty("vespa.test.credentials.root")}. */ + public SSLContext sslContext() { + try { + Path credentialsRoot = Path.of(System.getProperty("vespa.test.credentials.root")); + Path certificateFile = credentialsRoot.resolve("cert"); + Path privateKeyFile = credentialsRoot.resolve("key"); + + X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile))); + if (Instant.now().isBefore(certificate.getNotBefore().toInstant()) + || Instant.now().isAfter(certificate.getNotAfter().toInstant())) + throw new IllegalStateException("Certificate at '" + certificateFile + "' is valid between " + + certificate.getNotBefore() + " and " + certificate.getNotAfter() + " — not now."); + + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyFile))); + return new SslContextBuilder().withKeyStore(privateKey, certificate).build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public HttpRequest.Builder authenticated(HttpRequest.Builder request) { + return request; + } + + ApplicationId id = ApplicationId.from(requireNonBlankProperty("tenant"), + requireNonBlankProperty("application"), + getNonBlankProperty("instance").orElse("default")); + + URI endpoint = URI.create(requireNonBlankProperty("endpoint")); + Path privateKeyFile = Paths.get(requireNonBlankProperty("privateKeyFile")); + Optional<Path> certificateFile = getNonBlankProperty("certificateFile").map(Paths::get); + + ControllerHttpClient controller = certificateFile.isPresent() + ? ControllerHttpClient.withKeyAndCertificate(endpoint, privateKeyFile, certificateFile.get()) + : ControllerHttpClient.withSignatureKey(endpoint, privateKeyFile, id); + + static Optional<String> getNonBlankProperty(String name) { + return Optional.ofNullable(System.getProperty(name)).filter(value -> ! value.isBlank()); + } + + static String requireNonBlankProperty(String name) { + return getNonBlankProperty(name).orElseThrow(() -> new IllegalStateException("Missing required property '" + name + "'")); + } + +} diff --git a/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java b/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java new file mode 100644 index 00000000000..ff4bebce3ff --- /dev/null +++ b/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java @@ -0,0 +1,5 @@ +package ai.vespa.hosted.auth; + +public class AuthenticatorTest { + +} diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 8d5fb626789..9c3a28964ed 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -41,7 +41,6 @@ <dependencyManagement> <dependencies> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dependency-versions</artifactId> @@ -49,7 +48,6 @@ <type>pom</type> <scope>import</scope> </dependency> - </dependencies> </dependencyManagement> @@ -224,21 +222,9 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <groups>com.yahoo.vespa.tenant.cd.SystemTest</groups> - <excludedGroups /> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <groups>ai.vespa.hosted.cd.SystemTest</groups> + <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> </plugins> @@ -252,21 +238,9 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <groups>com.yahoo.vespa.tenant.cd.StagingTest</groups> - <excludedGroups /> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <groups>ai.vespa.hosted.cd.StagingTest</groups> + <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> </plugins> @@ -280,21 +254,9 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <groups>com.yahoo.vespa.tenant.cd.ProductionTest</groups> - <excludedGroups /> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <groups>ai.vespa.hosted.cd.ProductionTest</groups> + <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> </plugins> @@ -304,8 +266,38 @@ <build> <finalName>${project.artifactId}</finalName> - <plugins> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire_version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <endpoint>${endpoint}</endpoint> + <privateKeyFile>${privateKeyFile}</privateKeyFile> + <certificateFile>${certificateFile}</certificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${surefire_version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> @@ -369,25 +361,14 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> <excludedGroups> - com.yahoo.vespa.tenant.cd.SystemTest, - com.yahoo.vespa.tenant.cd.StagingTest, - com.yahoo.vespa.tenant.cd.ProductionTest + ai.vespa.hosted.cd.SystemTest, + ai.vespa.hosted.cd.StagingTest, + ai.vespa.hosted.cd.ProductionTest </excludedGroups> </configuration> </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/tenant-cd/pom.xml b/tenant-cd/pom.xml index 8907e56762c..7cc2c9a2d5b 100644 --- a/tenant-cd/pom.xml +++ b/tenant-cd/pom.xml @@ -5,6 +5,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> + <groupId>ai.vespa.hosted</groupId> <artifactId>tenant-cd</artifactId> <name>Hosted Vespa tenant CD</name> <description>Test library for hosted Vespa applications.</description> @@ -20,6 +21,36 @@ <dependencies> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-auth</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java new file mode 100644 index 00000000000..277632b74c7 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java @@ -0,0 +1,19 @@ +package ai.vespa.hosted.cd; + +/** + * A deployment of a Vespa application, which contains endpoints for document and metrics retrieval. + * + * @author jonmv + */ +public interface Deployment { + + /** Returns an Endpoint in the cluster with the "default" id. */ + Endpoint endpoint(); + + /** Returns an Endpoint in the cluster with the given id. */ + Endpoint endpoint(String id); + + /** Returns a {@link TestDeployment} representation of this, or throws if this is a production deployment. */ + TestDeployment asTestDeployment(); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java new file mode 100644 index 00000000000..dee13fdca13 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java @@ -0,0 +1,28 @@ +package ai.vespa.hosted.cd; + +import java.util.Set; + +/** + * An immutable report of the outcome of a {@link Feed} sent to a {@link TestEndpoint}. + * + * @author jonmv + */ +public class Digest { + + public Set<DocumentId> created() { + return null; + } + + public Set<DocumentId> updated() { + return null; + } + + public Set<DocumentId> deleted() { + return null; + } + + public Set<DocumentId> failed() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java new file mode 100644 index 00000000000..91adeded65c --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java @@ -0,0 +1,16 @@ +package ai.vespa.hosted.cd; + +/** + * A schema-less representation of a generic Vespa document. + * + * @author jonmv + */ +public class Document { + + + /** Returns a copy of this document, updated with the data in the given document. */ + public Document updatedBy(Document update) { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java new file mode 100644 index 00000000000..9aa8e80c977 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java @@ -0,0 +1,71 @@ +package ai.vespa.hosted.cd; + +import java.util.Arrays; +import java.util.List; + +/** + * Unique, immutable ID of a Vespa document, which contains information pertinent to its storage. + * + * @author jonmv + */ +public class DocumentId { + + private final String namespace; + private final String documentType; + private final String group; + private final Long number; + private final String userDefined; + + private DocumentId(String namespace, String documentType, String group, Long number, String userDefined) { + this.namespace = namespace; + this.documentType = documentType; + this.group = group; + this.number = number; + this.userDefined = userDefined; + } + + public static DocumentId of(String namespace, String documentType, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, null, requireNonEmpty(id)); + } + + public static DocumentId of(String namespace, String documentType, String group, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), requireNonEmpty(group), null, requireNonEmpty(id)); + } + + public static DocumentId of(String namespace, String documentType, long number, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, number, requireNonEmpty(id)); + } + + public static DocumentId ofValue(String value) { + List<String> parts = Arrays.asList(value.split(":")); + String id = String.join(":", parts.subList(4, parts.size())); + if ( parts.size() < 5 + || ! parts.get(0).equals("id") + || id.isEmpty() + || ! parts.get(3).matches("((n=\\d+)|(g=\\w+))?")) + throw new IllegalArgumentException("Document id must be on the form" + + " 'id:<namespace>:<document type>:n=<integer>|g=<name>|<empty>:<user defined id>'," + + " but was '" + value + "'."); + + if (parts.get(3).matches("n=\\d+")) + return of(parts.get(1), parts.get(2), Long.parseLong(parts.get(3).substring(2)), id); + if (parts.get(3).matches("g=\\w+")) + return of(parts.get(1), parts.get(2), parts.get(3).substring(2), id); + return of(parts.get(1), parts.get(2), id); + } + + public String asValue() { + return "id:" + namespace + ":" + documentType + ":" + grouper() + ":" + userDefined; + } + + private String grouper() { + return group != null ? group : number != null ? number.toString() : ""; + } + + private static String requireNonEmpty(String string) { + if (string.isEmpty()) + throw new IllegalArgumentException("The empty string is not allowed."); + return string; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java new file mode 100644 index 00000000000..8deca3cfb11 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java @@ -0,0 +1,9 @@ +package ai.vespa.hosted.cd; + +/** + * The Surefire configuration element <excludedGroups> requires a non-empty argument to reset another. + * This class serves that purpose. Without it, no tests run in the various integration test profiles. + * + * @author jonmv + */ +public interface EmptyGroup { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java new file mode 100644 index 00000000000..348fc329682 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -0,0 +1,21 @@ +package ai.vespa.hosted.cd; + +import ai.vespa.hosted.cd.metric.Metrics; + +/** + * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval. + * + * The endpoint translates {@link Query}s to {@link Search}s, and {@link Selection}s to {@link Visit}s. + * It also supplies {@link Metrics}. + * + * @author jonmv + */ +public interface Endpoint { + + Search search(Query query); + + Visit visit(Selection selection); + + Metrics metrics(); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java new file mode 100644 index 00000000000..e9a0a0aeff0 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java @@ -0,0 +1,25 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; +import java.util.Set; + +/** + * An immutable set of document feed / update / delete operations, which can be sent to a Vespa {@link TestEndpoint}. + * + * @author jonmv + */ +public class Feed { + + Map<DocumentId, Document> creations() { + return null; + } + + Map<DocumentId, Document> updates() { + return null; + } + + Set<DocumentId> deletions() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java new file mode 100644 index 00000000000..e6beb313d28 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java @@ -0,0 +1,31 @@ +package ai.vespa.hosted.cd; + +/** + * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical functional test is to feed some documents, optionally verifying that the documents have been processed + * as expected, and then to see that queries give the expected results. Another common use is to verify integration + * with external services. + * + * @author jonmv + */ +public interface FunctionalTest { + + // Want to feed some documents. + // Want to verify document processing and routing is as expected. + // Want to check recall on those documents. + // Want to verify queries give expected documents. + // Want to verify searchers. + // Want to verify updates. + // Want to verify deletion. + // May want to verify reprocessing. + // Must likely delete documents between tests. + // Must be able to feed documents, setting route. + // Must be able to search. + // Must be able to visit. + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index a756b665c1a..6cf5fb07f58 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -1,6 +1,23 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; -public class ProductionTest { +/** + * Tests that verify the health of production deployments of Vespa applications. + * + * These tests are typically run some time after deployment to a production zone, to ensure + * the deployment is still healthy and working as expected. When these tests fail, deployment + * of the tested change is halted until it succeeds, or is superseded by a remedying change. + * + * A typical production test is to verify that a set of metrics, measured by the Vespa + * deployment itself, are within specified parameters, or that some higher-level measure + * of quality, such as engagement among end users of the application, is as expected. + * + * @author jonmv + */ +public interface ProductionTest { + + // Want to verify metrics (Vespa). + // Want to verify external metrics (YAMAS, other). + // May want to verify search gives expected results. } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java new file mode 100644 index 00000000000..d421dc14322 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java @@ -0,0 +1,60 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Map.copyOf; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * An immutable query to send to a Vespa {@link Endpoint}, to receive a {@link Search}. + * + * @author jonmv + */ +public class Query { + + private final String rawQuery; + private final Map<String, String> parameters; + + private Query(String rawQuery, Map<String, String> parameters) { + this.rawQuery = rawQuery; + this.parameters = parameters; + } + + /** Creates a query with the given raw query part. */ + public static Query ofRaw(String rawQuery) { + if (rawQuery.isBlank()) + throw new IllegalArgumentException("Query can not be blank."); + + return new Query(rawQuery, + Stream.of(rawQuery.split("&")) + .map(pair -> pair.split("=")) + .collect(toUnmodifiableMap(pair -> pair[0], pair -> pair[1]))); + } + + /** Creates a query with the given name-value pairs. */ + public static Query ofParameters(Map<String, String> parameters) { + if (parameters.isEmpty()) + throw new IllegalArgumentException("Parameters can not be empty."); + + return new Query(parameters.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(joining("&")), + copyOf(parameters)); + } + + /** Returns a copy of this with the given name-value pair added, potentially overriding any current value. */ + public Query withParameter(String name, String value) { + return ofParameters(Stream.concat(parameters.entrySet().stream().filter(entry -> ! entry.getKey().equals(name)), + Stream.of(Map.entry(name, value))) + .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + /** Returns the raw string representation of this query. */ + public String rawQuery() { return rawQuery; } + + /** Returns the parameters of this query. */ + public Map<String, String> parameters() { return parameters; } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java new file mode 100644 index 00000000000..a6c1d188591 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java @@ -0,0 +1,24 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; + +/** + * The immutable result of sending a {@link Query} to a Vespa {@link Endpoint}. + * + * @author jonmv + */ +public class Search { + + // hits + // coverage + // searched + // full? + // results? + // resultsFull? + + /** Returns the documents that were returned as the result, with iteration order as returned. */ + Map<DocumentId, Document> documents() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java new file mode 100644 index 00000000000..158ae279cb6 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java @@ -0,0 +1,58 @@ +package ai.vespa.hosted.cd; + +/** + * A document selection expression, type and cluster, which can be used to visit an {@link Endpoint}. + * + * @author jonmv + */ +public class Selection { + + private final String selection; + private final String namespace; + private final String type; + private final String group; + private final String cluster; + private final int concurrency; + + private Selection(String selection, String namespace, String type, String group, String cluster, int concurrency) { + this.selection = selection; + this.namespace = namespace; + this.type = type; + this.group = group; + this.cluster = cluster; + this.concurrency = concurrency; + } + + /** Returns a new selection which will visit documents in the given cluster. */ + public static Selection in(String cluster) { + if (cluster.isBlank()) throw new IllegalArgumentException("Cluster name can not be blank."); + return new Selection(null, null, null, cluster, null, 1); + } + + /** Returns a new selection which will visit documents in the given namespace and of the given type. */ + public static Selection of(String namespace, String type) { + if (namespace.isBlank()) throw new IllegalArgumentException("Namespace can not be blank."); + if (type.isBlank()) throw new IllegalArgumentException("Document type can not be blank."); + return new Selection(null, namespace, type, null, null, 1); + } + + /** Returns a copy of this with the given selection criterion set. */ + public Selection matching(String selection) { + if (selection.isBlank()) throw new IllegalArgumentException("Selection can not be blank."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + + /** Returns a copy of this selection, with the group set to the specified value. Requires namespace and type to be set. */ + public Selection limitedTo(String group) { + if (namespace == null || type == null) throw new IllegalArgumentException("Namespace and type must be specified to set group."); + if (group.isBlank()) throw new IllegalArgumentException("Group name can not be blank."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + + /** Returns a copy of this, with concurrency set to the given positive value. */ + public Selection concurrently(int concurrency) { + if (concurrency < 1) throw new IllegalArgumentException("Concurrency must be a positive integer."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 789b9deadb0..ee2ee0add4c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -1,6 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; +/** + * @deprecated Use {@link UpgradeTest}. + */ +@Deprecated public class StagingTest { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java index 889acb8b9c4..6a8d1b4cbe4 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -1,6 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; +/** + * @deprecated use {@link FunctionalTest}. + */ +@Deprecated public class SystemTest { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java new file mode 100644 index 00000000000..36c14a38b37 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java @@ -0,0 +1,101 @@ +package ai.vespa.hosted.cd; + +import ai.vespa.hosted.api.ControllerHttpClient; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The place to obtain environment-dependent configuration for the current test run. + * + * If the system property 'vespa.test.config' is set, this class attempts to parse config + * from a JSON file at that location -- otherwise, attempts to access the config will return null. + * + * @author jvenstad + */ +public class TestConfig { + + private static TestConfig theConfig; + + private final ApplicationId application; + private final ZoneId zone; + private final SystemName system; + private final Map<ZoneId, Deployment> deployments; + + private TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Deployment> deployments) { + this.application = application; + this.zone = zone; + this.system = system; + this.deployments = Map.copyOf(deployments); + } + + /** Returns the config for this test, or null if it has not been provided. */ + public static synchronized TestConfig get() { + if (theConfig == null) { + String configPath = System.getProperty("vespa.test.config"); + theConfig = configPath != null ? fromFile(configPath) : fromController(); + } + return theConfig; + } + + /** Returns the full id of the application to be tested. */ + public ApplicationId application() { return application; } + + /** Returns the zone of the deployment to test. */ + public ZoneId zone() { return zone; } + + /** Returns an immutable view of all configured endpoints for each zone of the application to test. */ + public Map<ZoneId, Deployment> allDeployments() { return deployments; } + + /** Returns the deployment to test in this test runtime. */ + public Deployment deploymentToTest() { return deployments.get(zone); } + + /** Returns the system this is run against. */ + public SystemName system() { return system; } + + static TestConfig fromFile(String path) { + if (path == null) + return null; + + try { + return fromJson(Files.readAllBytes(Paths.get(path))); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); + } + } + + static TestConfig fromController() { + return null; + } + + static TestConfig fromJson(byte[] jsonBytes) { + Inspector config = new JsonDecoder().decode(new Slime(), jsonBytes).get(); + ApplicationId application = ApplicationId.fromSerializedForm(config.field("application").asString()); + ZoneId zone = ZoneId.from(config.field("zone").asString()); + SystemName system = SystemName.from(config.field("system").asString()); + Map<ZoneId, Deployment> endpoints = new HashMap<>(); + config.field("endpoints").traverse((ObjectTraverser) (zoneId, endpointArray) -> { + List<URI> uris = new ArrayList<>(); + endpointArray.traverse((ArrayTraverser) (__, uri) -> uris.add(URI.create(uri.asString()))); + endpoints.put(ZoneId.from(zoneId), null); // TODO jvenstad + }); + return new TestConfig(application, zone, system, endpoints); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java new file mode 100644 index 00000000000..3360c12e374 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java @@ -0,0 +1,14 @@ +package ai.vespa.hosted.cd; + +/** + * A deployment of a Vespa application, which also contains endpoints for document manipulation. + * + * @author jonmv + */ +public interface TestDeployment extends Deployment { + + TestEndpoint endpoint(); + + TestEndpoint endpoint(String id); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java new file mode 100644 index 00000000000..f6f8a722f19 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java @@ -0,0 +1,13 @@ +package ai.vespa.hosted.cd; + +/** + * An endpoint in a Vespa application {@link TestDeployment}, which also translates {@link Feed}s to {@link Digest}s. + * + * @author jonmv + */ +public interface TestEndpoint extends Endpoint { + + /** Sends the given Feed to this TestEndpoint, blocking until it is digested, and returns a feed report. */ + Digest digest(Feed feed); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java new file mode 100644 index 00000000000..32083fbd5f6 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java @@ -0,0 +1,23 @@ +package ai.vespa.hosted.cd; + +/** + * Tests that assert continuity of behaviour for Vespa application deployments, through upgrades. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical upgrade test is to do some operations against a test deployment prior to upgrade, like feed and + * search for some documents, perhaps recording some metrics from the deployment, and then to upgrade it, + * repeat the exercise, and compare the results from pre and post upgrade. + * + * TODO Split in platform upgrades and application upgrades? + * + * @author jonmv + */ +public interface UpgradeTest { + + // Want to verify documents are not damaged by upgrade. + // May want to verify metrics during upgrade. + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java new file mode 100644 index 00000000000..3bb2f59de97 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java @@ -0,0 +1,17 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; + +/** + * A stateful visit operation against a {@link Endpoint}. + * + * @author jonmv + */ +public class Visit { + + // Delegate to a blocking iterator, which can be used for iteration as visit is ongoing. + public Map<DocumentId, Document> documents() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java new file mode 100644 index 00000000000..618a004a571 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java @@ -0,0 +1,10 @@ +package ai.vespa.hosted.cd; + +/** + * A remote endpoint in a Vespa application {@link Deployment}, which translates {@link Selection}s to {@link Visit}s. + * + * @author jonmv + */ +public interface VisitEndpoint { + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java new file mode 100644 index 00000000000..e0d3787a21c --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -0,0 +1,85 @@ +package ai.vespa.hosted.cd.http; + +import ai.vespa.hosted.auth.Authenticator; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import ai.vespa.hosted.cd.Digest; +import ai.vespa.hosted.cd.Feed; +import ai.vespa.hosted.cd.Query; +import ai.vespa.hosted.cd.Search; +import ai.vespa.hosted.cd.Selection; +import ai.vespa.hosted.cd.TestEndpoint; +import ai.vespa.hosted.cd.Visit; +import ai.vespa.hosted.cd.metric.Metrics; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +import static java.util.Objects.requireNonNull; + +public class HttpEndpoint implements TestEndpoint { + + static final String metricsPath = "/state/v1/metrics"; + static final String documentApiPath = "/document/v1"; + static final String searchApiPath = "/search"; + + private final URI endpoint; + private final HttpClient client; + private final Authenticator authenticator; + + public HttpEndpoint(URI endpoint) { + this.endpoint = requireNonNull(endpoint); + this.authenticator = new Authenticator(); + this.client = HttpClient.newBuilder() + .sslContext(authenticator.sslContext()) + .connectTimeout(Duration.ofSeconds(5)) + .version(HttpClient.Version.HTTP_1_1) + .build(); + } + + @Override + public Digest digest(Feed feed) { + return null; + } + + @Override + public Search search(Query query) { + try { + URI target = endpoint.resolve(searchApiPath).resolve("?" + query.rawQuery()); + HttpRequest request = HttpRequest.newBuilder() + .timeout(Duration.ofSeconds(5)) + .uri(target) + .build(); + HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() / 100 != 2) // TODO consider allowing 504 if specified. + throw new RuntimeException("Non-OK status code " + response.statusCode() + " at " + target + + ", with response \n" + new String(response.body())); + + return toSearch(response.body()); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + static Search toSearch(byte[] body) { + Inspector rootObject = new JsonDecoder().decode(new Slime(), body).get(); + // TODO jvenstad + return new Search(); + } + + @Override + public Visit visit(Selection selection) { + return null; + } + + @Override + public Metrics metrics() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java new file mode 100644 index 00000000000..cb3c8e77a9a --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java @@ -0,0 +1,87 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * A set of statistics for a metric, for points over a space with named dimensions of arbitrary type. + * + * @author jonmv + */ +public class Metric { + + private final Map<Map<String, ?>, Statistic> statistics; + + private Metric(Map<Map<String, ?>, Statistic> statistics) { + this.statistics = statistics; + } + + /** Creates a new Metric with a copy of the given data. */ + public static Metric of(Map<Map<String, ?>, Statistic> data) { + if (data.isEmpty()) + throw new IllegalArgumentException("No data given."); + + Map<Map<String, ?>, Statistic> copies = new HashMap<>(); + Set<String> dimensions = data.keySet().iterator().next().keySet(); + data.forEach((point, statistic) -> { + if ( ! point.keySet().equals(dimensions)) + throw new IllegalArgumentException("Given data has inconsistent dimensions: '" + dimensions + "' vs '" + point.keySet() + "'."); + + copies.put(copyOf(point), statistic); + }); + + return new Metric(copyOf(copies)); + } + + /** Returns a Metric view of the subset of points in the given hyperplane; its dimensions must be a subset of those of this Metric. */ + public Metric at(Map<String, ?> hyperplane) { + return new Metric(statistics.keySet().stream() + .filter(point -> point.entrySet().containsAll(hyperplane.entrySet())) + .collect(toUnmodifiableMap(point -> point, statistics::get))); + } + + /** Returns a version of this where statistics along the given hyperspace are aggregated. This does not preserve last, 95 and 99 percentile values. */ + public Metric collapse(Set<String> hyperspace) { + return new Metric(statistics.keySet().stream() + .collect(toUnmodifiableMap(point -> point.keySet().stream() + .filter(dimension -> ! hyperspace.contains(dimension)) + .collect(toUnmodifiableMap(dimension -> dimension, point::get)), + statistics::get, + Statistic::mergedWith))); + } + + /** Returns a collapsed version of this, with all statistics aggregated. This does not preserve last, 95 and 99 percentile values. */ + public Metric collapse() { + return collapse(statistics.keySet().iterator().next().keySet()); + } + + /** If this Metric contains a single point, returns the Statistic of that point; otherwise, throws an exception. */ + public Statistic statistic() { + if (statistics.size() == 1) + return statistics.values().iterator().next(); + + if (statistics.isEmpty()) + throw new NoSuchElementException("This Metric has no data."); + + throw new IllegalStateException("This Metric has more than one point of data."); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<Map<String, ?>, Statistic> asMap() { + return statistics; + } + + @Override + public String toString() { + return new StringJoiner(", ", Metric.class.getSimpleName() + "[", "]") + .add("statistics=" + statistics) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java new file mode 100644 index 00000000000..3aa5a126745 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java @@ -0,0 +1,73 @@ +package ai.vespa.hosted.cd.metric; + +import ai.vespa.hosted.cd.Endpoint; + +import java.time.Instant; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; + +/** + * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions. + * + * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics-health-format.html">metrics</a> + * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in + * that endpoint, and the time interval from which these metrics were sampled. + * + * Each metric is indexed by a name, and, optionally, along a custom set of dimensions, given by a {@code Map<String, String>}. + * + * @author jonmv + */ +public class Metrics { + + private final Instant start, end; + private final Map<String, Metric> metrics; + + private Metrics(Instant start, Instant end, Map<String, Metric> metrics) { + this.start = start; + this.end = end; + this.metrics = metrics; + } + + public static Metrics of(Instant start, Instant end, Map<String, Metric> metrics) { + if ( ! start.isBefore(end)) + throw new IllegalArgumentException("Given time interval must be positive: '" + start + "' to '" + end + "'."); + + return new Metrics(start, end, copyOf(metrics)); + } + + /** Returns the start of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ + public Instant start() { + return start; + } + + /** Returns the end of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ + public Instant end() { + return end; + } + + /** Returns the metric with the given name, or throws a NoSuchElementException if no such Metric is known. */ + public Metric get(String name) { + if ( ! metrics.containsKey(name)) + throw new NoSuchElementException("No metric with name '" + name + "'."); + + return metrics.get(name); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<String, Metric> asMap() { + return metrics; + } + + @Override + public String toString() { + return new StringJoiner(", ", Metrics.class.getSimpleName() + "[", "]") + .add("start=" + start) + .add("end=" + end) + .add("metrics=" + metrics) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java new file mode 100644 index 00000000000..ea771ca5dd9 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java @@ -0,0 +1,44 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * Used to easily generate points for a pre-defined space. + * + * @author jonmv + */ +public class Space { + + private final List<String> dimensions; + + private Space(List<String> dimensions) { + this.dimensions = dimensions; + } + + /** Creates a new space with the given named dimensions, in order. */ + public static Space of(List<String> dimensions) { + if (Set.copyOf(dimensions).size() != dimensions.size()) + throw new IllegalArgumentException("Duplicated dimension names in '" + dimensions + "'."); + + return new Space(List.copyOf(dimensions)); + } + + /** Returns a point in this space, with the given values along each dimensions, in order. */ + public Map<String, ?> at(List<?> values) { + if (dimensions.size() != values.size()) + throw new IllegalArgumentException("This space has " + dimensions.size() + " dimensions, but " + values.size() + " were given."); + + return IntStream.range(0, dimensions.size()).boxed().collect(toUnmodifiableMap(dimensions::get, values::get)); + } + + /** Returns a point in this space, with the given values along each dimensions, in order. */ + public Map<String, ?> at(Object... values) { + return at(List.of(values)); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java new file mode 100644 index 00000000000..fc52900bdac --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java @@ -0,0 +1,68 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; + +/** + * Known statistic about a metric, at a certain point. + * + * @author jonmv + */ +public class Statistic { + + private final Map<Type, Double> data; + + /** Creates a new Statistic with a copy of the given data. */ + private Statistic(Map<Type, Double> data) { + this.data = data; + } + + public static Statistic of(Map<Type, Double> data) { + return new Statistic(copyOf(data)); + } + + /** Returns the value of the given type, or throws a NoSuchElementException if this isn't known. */ + public double get(Type key) { + if ( ! data.containsKey(key)) + throw new NoSuchElementException("No value with key '" + key + "' is known."); + + return data.get(key); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<Type, Double> asMap() { + return data; + } + + Statistic mergedWith(Statistic other) { + if (data.keySet().equals(other.data.keySet())) + throw new IllegalArgumentException("Incompatible key sets '" + data.keySet() + "' and '" + other.data.keySet() + "'."); + + Map<Type, Double> merged = new HashMap<>(); + double n1 = get(Type.count), n2 = other.get(Type.count); + for (Type type : data.keySet()) switch (type) { + case count: merged.put(type, n1 + n2); break; + case rate: merged.put(type, get(Type.rate) + other.get(Type.rate)); break; + case max: merged.put(type, Math.max(get(Type.max), other.get(Type.max))); break; + case min: merged.put(type, Math.min(get(Type.min), other.get(Type.min))); break; + case average: merged.put(type, (n1 * get(Type.average) + n2 * other.get(Type.average)) / (n1 + n2)); break; + case last: + case percentile95: + case percentile99: break; + default: throw new IllegalArgumentException("Unexpected type '" + type + "'."); + } + return of(merged); + } + + @Override + public String toString() { + return new StringJoiner(", ", Statistic.class.getSimpleName() + "[", "]") + .add("data=" + data) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java new file mode 100644 index 00000000000..d48b4566f6d --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java @@ -0,0 +1,32 @@ +package ai.vespa.hosted.cd.metric; + +/** + * Known statistic types. + */ +public enum Type { + + /** 95th percentile measurement. */ + percentile95, + + /** 99th percentile measurement. */ + percentile99, + + /** Average over all measurements. */ + average, + + /** Number of measurements. */ + count, + + /** Last measurement. */ + last, + + /** Maximum measurement. */ + max, + + /** Minimum measurement. */ + min, + + /** Number of measurements per second. */ + rate; + +} diff --git a/vbench/src/vbench/http/benchmark_headers.h b/vbench/src/vbench/http/benchmark_headers.h index 92c4a05271b..04b29813760 100644 --- a/vbench/src/vbench/http/benchmark_headers.h +++ b/vbench/src/vbench/http/benchmark_headers.h @@ -5,6 +5,7 @@ #include <vbench/core/string.h> #include <vespa/vespalib/locale/c.h> +#include <cerrno> namespace vbench { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 0803c346b0a..c830f89d5a9 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using namespace vdslib; diff --git a/vespalib/src/vespa/vespalib/btree/btree.hpp b/vespalib/src/vespa/vespalib/btree/btree.hpp index 928d8d6cfcd..7bba2e936f3 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.hpp +++ b/vespalib/src/vespa/vespalib/btree/btree.hpp @@ -4,8 +4,7 @@ #include "btree.h" -namespace search { -namespace btree { +namespace search::btree { template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, class AggrCalcT> @@ -24,7 +23,4 @@ BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::~BTree() _alloc.clearHoldLists(); } - -} // namespace search::btree -} // namespace search - +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp b/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp index f307c474f90..c00d2342eed 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp @@ -6,16 +6,11 @@ #include "btreeinserter.hpp" #include "btreenode.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.btree.btreeinserter"); - namespace search::btree { template class BTreeInserter<uint32_t, uint32_t, NoAggregated>; template class BTreeInserter<uint32_t, BTreeNoLeafData, NoAggregated>; template class BTreeInserter<uint32_t, int32_t, MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.h b/vespalib/src/vespa/vespalib/btree/btreeinserter.h index a3fa2916a88..a66a7bc3f92 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.h +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.h @@ -10,11 +10,7 @@ #include "minmaxaggrcalc.h" #include "btreeiterator.h" -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, @@ -32,12 +28,9 @@ public: TraitsT::INTERNAL_SLOTS, TraitsT::LEAF_SLOTS, AggrCalcT> Aggregator; - typedef BTreeIterator<KeyT, DataT, AggrT, - CompareT, TraitsT> Iterator; - typedef BTreeInternalNode<KeyT, AggrT, TraitsT::INTERNAL_SLOTS> - InternalNodeType; - typedef BTreeLeafNode<KeyT, DataT, AggrT, TraitsT::LEAF_SLOTS> - LeafNodeType; + typedef BTreeIterator<KeyT, DataT, AggrT, CompareT, TraitsT> Iterator; + typedef BTreeInternalNode<KeyT, AggrT, TraitsT::INTERNAL_SLOTS> InternalNodeType; + typedef BTreeLeafNode<KeyT, DataT, AggrT, TraitsT::LEAF_SLOTS> LeafNodeType; typedef KeyT KeyType; typedef DataT DataType; typedef typename InternalNodeType::RefPair InternalNodeTypeRefPair; @@ -49,19 +42,12 @@ private: public: static void - insert(BTreeNode::Ref &root, - Iterator &itr, - const KeyType &key, const DataType &data, - const AggrCalcT &aggrCalc); + insert(BTreeNode::Ref &root, Iterator &itr, const KeyType &key, const DataType &data, const AggrCalcT &aggrCalc); }; extern template class BTreeInserter<uint32_t, uint32_t, NoAggregated>; extern template class BTreeInserter<uint32_t, BTreeNoLeafData, NoAggregated>; extern template class BTreeInserter<uint32_t, int32_t, MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; - -} // namespace search::btree -} // namespace search + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp index d1da94c1b17..b24874088a2 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp @@ -7,8 +7,7 @@ #include "btreeiterator.hpp" #include <vespa/vespalib/stllike/asciistream.h> -namespace search { -namespace btree { +namespace search::btree { namespace { @@ -178,7 +177,4 @@ insert(BTreeNode::Ref &root, } } - -} // namespace search::btree -} // namespace search - +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index b26f249c51b..13d41ef61f3 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -9,8 +9,6 @@ namespace search::btree { -#define STRICT_BTREE_ITERATOR_SEEK - template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: @@ -81,10 +79,7 @@ operator=(const BTreeIteratorBase &other) template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> -BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -~BTreeIteratorBase() -{ -} +BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>::~BTreeIteratorBase() = default; template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> @@ -674,11 +669,7 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: binarySeek(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && comp(lnode->getKey(lidx), key)); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; if (lidx < lnode->validSlots()) { if (!comp(lnode->getKey(lidx), key)) { _leaf.setIdx(lidx); @@ -723,11 +714,7 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: linearSeek(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && comp(lnode->getKey(lidx), key)); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; if (lidx < lnode->validSlots()) { if (!comp(lnode->getKey(lidx), key)) { _leaf.setIdx(lidx); @@ -792,11 +779,7 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: binarySeekPast(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && !comp(key, lnode->getKey(lidx))); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; if (lidx < lnode->validSlots()) { if (comp(key, lnode->getKey(lidx))) { _leaf.setIdx(lidx); @@ -841,11 +824,8 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: linearSeekPast(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && !comp(key, lnode->getKey(lidx))); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; + if (lidx < lnode->validSlots()) { if (comp(key, lnode->getKey(lidx))) { _leaf.setIdx(lidx); diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.cpp b/vespalib/src/vespa/vespalib/btree/btreeremover.cpp index 2322eebf784..f5ada77fed6 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.cpp @@ -11,8 +11,6 @@ namespace search::btree { template class BTreeRemover<uint32_t, uint32_t, NoAggregated>; template class BTreeRemover<uint32_t, BTreeNoLeafData, NoAggregated>; template class BTreeRemover<uint32_t, int32_t, MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.h b/vespalib/src/vespa/vespalib/btree/btreeremover.h index 87355aa4ce7..bbb825d6299 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.h +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.h @@ -10,11 +10,7 @@ #include "minmaxaggrcalc.h" #include "btreeiterator.h" -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, @@ -82,23 +78,15 @@ public: typedef DataT DataType; typedef typename InternalNodeType::RefPair InternalNodeTypeRefPair; typedef typename LeafNodeType::RefPair LeafNodeTypeRefPair; - typedef BTreeIterator<KeyT, DataT, AggrT, - CompareT, TraitsT> Iterator; + typedef BTreeIterator<KeyT, DataT, AggrT, CompareT, TraitsT> Iterator; static void - remove(BTreeNode::Ref &root, - Iterator &itr, - const AggrCalcT &aggrCalc); + remove(BTreeNode::Ref &root, Iterator &itr, const AggrCalcT &aggrCalc); }; extern template class BTreeRemover<uint32_t, uint32_t, NoAggregated>; extern template class BTreeRemover<uint32_t, BTreeNoLeafData, NoAggregated>; -extern template class BTreeRemover<uint32_t, int32_t, - MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; - -} // namespace search::btree -} // namespace search +extern template class BTreeRemover<uint32_t, int32_t, MinMaxAggregated, + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp index c304ea13016..2281fd99f6d 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp @@ -6,11 +6,7 @@ #include "btreerootbase.hpp" #include <vespa/vespalib/stllike/asciistream.h> -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> @@ -179,7 +175,4 @@ remove(BTreeNode::Ref &root, ++itr; } - -} // namespace search::btree -} // namespace search - +} diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 9b04724b601..758922aec6d 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "databuffer.h" +#include <algorithm> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/benchmark_timer.h b/vespalib/src/vespa/vespalib/util/benchmark_timer.h index 8d4147907fc..8e0821d6127 100644 --- a/vespalib/src/vespa/vespalib/util/benchmark_timer.h +++ b/vespalib/src/vespa/vespalib/util/benchmark_timer.h @@ -3,6 +3,7 @@ #include <chrono> #include <functional> +#include <algorithm> namespace vespalib { diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index 1723f35e432..802514285b6 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -3,6 +3,7 @@ #include <vespa/fastos/time.h> #include <vespa/fastos/thread.h> #include <vespa/log/bufferedlogger.h> +#include <array> #include <iostream> #include <thread> #include <chrono> diff --git a/vespalog/src/vespa/log/log_message.h b/vespalog/src/vespa/log/log_message.h index 832b5f6d47d..ac00e4237ac 100644 --- a/vespalog/src/vespa/log/log_message.h +++ b/vespalog/src/vespa/log/log_message.h @@ -4,6 +4,7 @@ #include "log.h" #include <string_view> +#include <string> namespace ns_log { |