diff options
232 files changed, 1501 insertions, 1472 deletions
diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp index 6784f0793e9..4d9eac41af6 100644 --- a/config/src/vespa/config/common/configparser.cpp +++ b/config/src/vespa/config/common/configparser.cpp @@ -14,7 +14,7 @@ void ConfigParser::throwNoDefaultValue(const vespalib::stringref & key) { } vespalib::string -ConfigParser::deQuote(const vespalib::stringref & source) +ConfigParser::deQuote(const vespalib::string & source) { const char *src = source.c_str(); const char *s = src; diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h index 2f6cdd9cf39..77b5bf7ddaa 100644 --- a/config/src/vespa/config/common/configparser.h +++ b/config/src/vespa/config/common/configparser.h @@ -23,7 +23,7 @@ private: static std::vector<vsvector> splitArray( const vsvector & config); static std::map<vespalib::string, vsvector> splitMap( const vsvector & config); - static vespalib::string deQuote(const vespalib::stringref & source); + static vespalib::string deQuote(const vespalib::string & source); static void throwNoDefaultValue(const vespalib::stringref & key); template<typename T> diff --git a/container-core/pom.xml b/container-core/pom.xml index d8473c8b541..d51930b5059 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -95,9 +95,19 @@ <version>${project.version}</version> <exclusions> <exclusion> + <!-- Pulled in by language-detector in scope compile --> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> <groupId>log4j</groupId> <artifactId>log4j</artifactId> </exclusion> + <exclusion> + <!-- Pulled in by language-detector in scope compile --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java deleted file mode 100644 index 967af1c553f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athenz; - -/** - * @author bjorncs - */ -public class InvalidTokenException extends RuntimeException { - public InvalidTokenException(String message) { - super(message); - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java index e8bc16ca271..3630748b10a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzPublicKey; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import java.util.List; @@ -33,8 +31,4 @@ public interface ZmsClient { List<AthenzDomain> getDomainList(String prefix); - AthenzPublicKey getPublicKey(AthenzService service, String keyId); - - List<AthenzPublicKey> getPublicKeys(AthenzService service); - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java deleted file mode 100644 index b3dc9fd4fe1..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.athenz; - -import com.yahoo.vespa.athenz.api.AthenzService; - -import java.security.PublicKey; -import java.util.Optional; - -/** - * @author bjorncs - */ -public interface ZmsKeystore { - - Optional<PublicKey> getPublicKey(AthenzService service, String keyId); - - default void preloadKeys(AthenzService service) { /* Default implementation is noop */ } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java deleted file mode 100644 index 4dcca519058..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; - -import com.yahoo.athenz.auth.token.PrincipalToken; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.NToken; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; - -import java.security.PublicKey; -import java.time.Duration; -import java.util.Optional; -import java.util.logging.Logger; - -import static com.yahoo.vespa.athenz.utils.AthenzIdentities.ZMS_ATHENZ_SERVICE; - - -/** - * Validates the content of an NToken: - * 1) Verifies that the token is signed by the sys.auth.zms service (by validating the signature) - * 2) Verifies that the token is not expired - * - * @author bjorncs - */ -// TODO Move to vespa-athenz -class NTokenValidator { - - // Max allowed skew in token timestamp (only for creation, not expiry timestamp) - private static final long ALLOWED_TIMESTAMP_OFFSET = Duration.ofMinutes(5).getSeconds(); - - private static final Logger log = Logger.getLogger(NTokenValidator.class.getName()); - - private final ZmsKeystore keystore; - - NTokenValidator(ZmsKeystore keystore) { - this.keystore = keystore; - } - - void preloadPublicKeys() { - keystore.preloadKeys(ZMS_ATHENZ_SERVICE); - } - - AthenzPrincipal validate(NToken token) throws InvalidTokenException { - PrincipalToken principalToken = new PrincipalToken(token.getRawToken()); - PublicKey zmsPublicKey = getPublicKey(principalToken.getKeyId()) - .orElseThrow(() -> new InvalidTokenException("NToken has an unknown keyId")); - validateSignatureAndExpiration(principalToken, zmsPublicKey); - return new AthenzPrincipal( - AthenzIdentities.from( - new AthenzDomain(principalToken.getDomain()), - principalToken.getName()), - token); - } - - private Optional<PublicKey> getPublicKey(String keyId) throws InvalidTokenException { - try { - return keystore.getPublicKey(ZMS_ATHENZ_SERVICE, keyId); - } catch (Exception e) { - logDebug(e.getMessage()); - throw new InvalidTokenException("Failed to retrieve public key"); - } - } - - private static void validateSignatureAndExpiration(PrincipalToken token, - PublicKey zmsPublicKey) throws InvalidTokenException { - StringBuilder errorMessageBuilder = new StringBuilder(); - if (!token.validate(zmsPublicKey, (int) ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { - String message = "NToken is expired or has invalid signature: " + errorMessageBuilder.toString(); - logDebug(message); - throw new InvalidTokenException(message); - } - } - - private static void logDebug(String message) { - log.log(LogLevel.DEBUG, "Failed to validate NToken: " + message); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java index b801c038bd8..26cd9f2e9b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java @@ -4,19 +4,19 @@ package com.yahoo.vespa.hosted.controller.athenz.filter; import com.google.inject.Inject; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.security.athenz.AthenzPrincipalFilter; +import com.yahoo.jdisc.http.filter.security.athenz.AthenzPrincipalFilterConfig; import com.yahoo.jdisc.http.filter.security.cors.CorsFilterConfig; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import com.yahoo.yolean.chain.After; import java.security.Principal; import java.util.Optional; -import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Stream; @@ -38,13 +38,10 @@ public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { private final String principalHeaderName; @Inject - public UserAuthWithAthenzPrincipalFilter(ZmsKeystore zmsKeystore, - Executor executor, - AthenzConfig athenzConfig, - CorsFilterConfig corsConfig) { - super(zmsKeystore, executor, athenzConfig, corsConfig); + public UserAuthWithAthenzPrincipalFilter(AthenzPrincipalFilterConfig filterConfig, AthenzConfig athenzConfig, CorsFilterConfig corsConfig) { + super(filterConfig, corsConfig); this.userAuthenticationPassThruAttribute = athenzConfig.userAuthenticationPassThruAttribute(); - this.principalHeaderName = athenzConfig.principalHeaderName(); + this.principalHeaderName = filterConfig.principalHeaderName(); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java index 67191d4c09d..6179d9891fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java @@ -1,22 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.impl; -import com.yahoo.athenz.auth.util.Crypto; import com.yahoo.athenz.zms.DomainList; import com.yahoo.athenz.zms.ProviderResourceGroupRoles; -import com.yahoo.athenz.zms.PublicKeyEntry; -import com.yahoo.athenz.zms.ServiceIdentity; import com.yahoo.athenz.zms.Tenancy; import com.yahoo.athenz.zms.TenantRoleAction; import com.yahoo.athenz.zms.ZMSClient; import com.yahoo.athenz.zms.ZMSClientException; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzPublicKey; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; @@ -130,28 +126,6 @@ public class ZmsClientImpl implements ZmsClient { }); } - @Override - public AthenzPublicKey getPublicKey(AthenzService service, String keyId) { - log("getPublicKeyEntry(domain=%s, service=%s, keyId=%s)", service.getDomain().getName(), service.getName(), keyId); - return getOrThrow(() -> { - PublicKeyEntry entry = zmsClient.getPublicKeyEntry(service.getDomain().getName(), service.getName(), keyId); - return fromYbase64EncodedKey(entry.getKey(), keyId); - }); - } - - @Override - public List<AthenzPublicKey> getPublicKeys(AthenzService service) { - log("getServiceIdentity(domain=%s, service=%s)", service.getDomain().getName(), service.getName()); - return getOrThrow(() -> { - ServiceIdentity serviceIdentity = zmsClient.getServiceIdentity(service.getDomain().getName(), service.getName()); - return toAthenzPublicKeys(serviceIdentity.getPublicKeys()); - }); - } - - private static AthenzPublicKey fromYbase64EncodedKey(String encodedKey, String keyId) { - return new AthenzPublicKey(Crypto.loadPublicKey(Crypto.ybase64DecodeString(encodedKey)), keyId); - } - private static List<TenantRoleAction> createTenantRoleActions() { return Arrays.stream(ApplicationAction.values()) .map(action -> new TenantRoleAction().setAction(action.name()).setRole(action.roleName)) @@ -162,12 +136,6 @@ public class ZmsClientImpl implements ZmsClient { return domains.stream().map(AthenzDomain::new).collect(toList()); } - private static List<AthenzPublicKey> toAthenzPublicKeys(List<PublicKeyEntry> publicKeys) { - return publicKeys.stream() - .map(entry -> fromYbase64EncodedKey(entry.getKey(), entry.getId())) - .collect(toList()); - } - private boolean hasAccess(String action, String resource, AthenzIdentity identity) { log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, identity); return getOrThrow( diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java deleted file mode 100644 index 4b194651439..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.impl; - -import com.google.inject.Inject; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.athenz.api.AthenzPublicKey; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; - -import java.security.PublicKey; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Logger; - -/** - * Downloads and caches public keys for Athens services. - * - * @author bjorncs - */ -public class ZmsKeystoreImpl implements ZmsKeystore { - private static final Logger log = Logger.getLogger(ZmsKeystoreImpl.class.getName()); - - private final Map<FullKeyId, PublicKey> cachedKeys = new ConcurrentHashMap<>(); - private final AthenzClientFactory athenzClientFactory; - - @Inject - public ZmsKeystoreImpl(AthenzClientFactory factory) { - this.athenzClientFactory = factory; - } - - @Override - public Optional<PublicKey> getPublicKey(AthenzService service, String keyId) { - FullKeyId fullKeyId = new FullKeyId(service, keyId); - PublicKey cachedKey = cachedKeys.get(fullKeyId); - if (cachedKey != null) { - return Optional.of(cachedKey); - } - Optional<PublicKey> downloadedKey = downloadPublicKey(fullKeyId); - downloadedKey.ifPresent(key -> { - log.log(LogLevel.INFO, "Adding key " + fullKeyId + " to the cache"); - cachedKeys.put(fullKeyId, key); - }); - return downloadedKey; - } - - @Override - public void preloadKeys(AthenzService service) { - try { - log.log(LogLevel.INFO, "Downloading keys for " + service); - List<AthenzPublicKey> publicKeys = athenzClientFactory.createZmsClientWithServicePrincipal() - .getPublicKeys(service); - for (AthenzPublicKey publicKey : publicKeys) { - FullKeyId fullKeyId = new FullKeyId(service, publicKey.getKeyId()); - log.log(LogLevel.DEBUG, "Adding key " + fullKeyId + " to the cache"); - cachedKeys.put(fullKeyId, publicKey.getPublicKey()); - } - log.log(LogLevel.INFO, "Successfully downloaded keys for " + service); - } catch (ZmsException e) { - log.log(LogLevel.WARNING, "Failed to download keys for " + service + ": " + e.getMessage()); - } - } - - private Optional<PublicKey> downloadPublicKey(FullKeyId fullKeyId) { - try { - log.log(LogLevel.INFO, "Downloading key " + fullKeyId); - AthenzPublicKey publicKey = athenzClientFactory.createZmsClientWithServicePrincipal() - .getPublicKey(fullKeyId.service, fullKeyId.keyId); - return Optional.of(publicKey.getPublicKey()); - } catch (ZmsException e) { - if (e.getCode() == 404) { // Key does not exist - log.log(LogLevel.INFO, "Key " + fullKeyId + " not found"); - return Optional.empty(); - } - String msg = String.format("Unable to retrieve public key from Athens (%s): %s", fullKeyId, e.getMessage()); - throw createException(msg, e); - } - } - - private static RuntimeException createException(String message, Exception cause) { - log.log(LogLevel.ERROR, message); - return new RuntimeException(message, cause); - } - - private static class FullKeyId { - private final AthenzService service; - private final String keyId; - - private FullKeyId(AthenzService service, String keyId) { - this.service = service; - this.keyId = keyId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FullKeyId fullKeyId1 = (FullKeyId) o; - return Objects.equals(service, fullKeyId1.service) && - Objects.equals(keyId, fullKeyId1.keyId); - } - - @Override - public int hashCode() { - return Objects.hash(service, keyId); - } - - @Override - public String toString() { - return "FullKeyId{" + - "service=" + service + - ", keyId='" + keyId + '\'' + - '}'; - } - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index 3ee2655108a..5e8674ce637 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -5,8 +5,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzPublicKey; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; @@ -96,16 +94,6 @@ public class ZmsClientMock implements ZmsClient { return new ArrayList<>(athenz.domains.keySet()); } - @Override - public AthenzPublicKey getPublicKey(AthenzService service, String keyId) { - throw new UnsupportedOperationException(); - } - - @Override - public List<AthenzPublicKey> getPublicKeys(AthenzService service) { - throw new UnsupportedOperationException(); - } - private AthenzDbMock.Domain getDomainOrThrow(AthenzDomain domainName, boolean verifyVespaTenant) { AthenzDbMock.Domain domain = Optional.ofNullable(athenz.domains.get(domainName)) .orElseThrow(() -> zmsException(400, "Domain '%s' not found", domainName)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java deleted file mode 100644 index 40b38254dda..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; - -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.security.NoSuchAlgorithmException; - -/** - * @author bjorncs - */ -public class AthenzTestUtils { - public static KeyPair generateRsaKeypair() { - try { - KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); - keyGen.initialize(512); - return keyGen.genKeyPair(); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java index d54fd2124f0..ce81e4e6dbd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java @@ -43,10 +43,7 @@ public class ZipBuilderTest { // Add the zipped data from zip1 to zip2 zipBuilder2.add(zipBuilder1.toByteArray()); - System.out.println(zipBuilder1.toByteArray().length); - System.out.println(zipBuilder2.toByteArray().length); Map<String, String> actual = unzipToMap(zipBuilder2.toByteArray()); - assertEquals(expected, actual); } } diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index c4cdc16c3ba..3ccb5aa714b 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -195,7 +195,7 @@ void deserialize(const ByteBuffer &buffer, T &value) { CPPUNIT_ASSERT_EQUAL(size_t(3), value2.getValueRef().size()); // Zero termination - CPPUNIT_ASSERT(*(value2.getValueRef().c_str() + value2.getValueRef().size()) == '\0'); + CPPUNIT_ASSERT(*(value2.getValueRef().data() + value2.getValueRef().size()) == '\0'); } } @@ -226,7 +226,7 @@ PrimitiveFieldValueTest::testRaw() value.toXml(" ")); value.setValue("grmpf", 4); - CPPUNIT_ASSERT(strncmp("grmpf", value.getValueRef().c_str(), + CPPUNIT_ASSERT(strncmp("grmpf", value.getValueRef().data(), value.getValueRef().size()) == 0); } diff --git a/document/src/vespa/document/base/documentid.cpp b/document/src/vespa/document/base/documentid.cpp index d2eae01922d..c3ba8fea29d 100644 --- a/document/src/vespa/document/base/documentid.cpp +++ b/document/src/vespa/document/base/documentid.cpp @@ -18,7 +18,7 @@ DocumentId::DocumentId() DocumentId::DocumentId(vespalib::stringref id) : Printable(), _globalId(), - _id(IdString::createIdString(id.c_str(), id.size()).release()) + _id(IdString::createIdString(id.data(), id.size()).release()) { } diff --git a/document/src/vespa/document/base/documentid.h b/document/src/vespa/document/base/documentid.h index 4611de73741..a4b01cdad82 100644 --- a/document/src/vespa/document/base/documentid.h +++ b/document/src/vespa/document/base/documentid.h @@ -41,10 +41,15 @@ public: * Parse the given document identifier given as string, and create an * identifier object from it. * + * Precondition: `id` MUST be null-terminated. + * * @throws IdParseException If the identifier given is invalid. */ explicit DocumentId(vespalib::stringref id); + /** + * Precondition: `id` MUST be null-terminated. + */ void set(vespalib::stringref id); /** diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp index 5a3fe6c1935..578294df7b0 100644 --- a/document/src/vespa/document/base/field.cpp +++ b/document/src/vespa/document/base/field.cpp @@ -78,7 +78,7 @@ Field::calculateIdV7() ost << getName(); ost << _dataType->getId(); - int newId = vespalib::BobHash::hash(ost.str().c_str(), ost.str().length(), 0); + int newId = vespalib::BobHash::hash(ost.str().data(), ost.str().length(), 0); // Highest bit is reserved to tell 7-bit id's from 31-bit ones if (newId < 0) newId = -newId; validateId(newId); @@ -91,7 +91,7 @@ Field::validateId(int newId) { throw vespalib::IllegalArgumentException(vespalib::make_string( "Attempt to set the id of %s to %d failed, values from " "100 to 127 are reserved for internal use", - getName().c_str(), newId)); + getName().data(), newId)); } if ((newId & 0x80000000) != 0) // Highest bit must not be set @@ -99,7 +99,7 @@ Field::validateId(int newId) { throw vespalib::IllegalArgumentException(vespalib::make_string( "Attempt to set the id of %s to %d" " failed, negative id values are illegal", - getName().c_str(), newId)); + getName().data(), newId)); } } diff --git a/document/src/vespa/document/base/fieldpath.cpp b/document/src/vespa/document/base/fieldpath.cpp index 6625f9ae2a5..d1603ca5c09 100644 --- a/document/src/vespa/document/base/fieldpath.cpp +++ b/document/src/vespa/document/base/fieldpath.cpp @@ -139,7 +139,7 @@ FieldPathEntry::visitMembers(vespalib::ObjectVisitor &visitor) const vespalib::string FieldPathEntry::parseKey(vespalib::stringref & key) { vespalib::string v; - const char *c = key.c_str(); + const char *c = key.data(); const char *e = c + key.size(); for(;(c < e) && isspace(c[0]); c++); if ((c < e) && (c[0] == '{')) { @@ -156,7 +156,8 @@ vespalib::string FieldPathEntry::parseKey(vespalib::stringref & key) if ((c < e) && (c[0] == '"')) { c++; } else { - throw IllegalArgumentException(make_string("Escaped key '%s' is incomplete. No matching '\"'", key.c_str()), VESPA_STRLOC); + throw IllegalArgumentException(make_string("Escaped key '%s' is incomplete. No matching '\"'", + vespalib::string(key).c_str()), VESPA_STRLOC); } } else { const char * start = c; @@ -169,10 +170,12 @@ vespalib::string FieldPathEntry::parseKey(vespalib::stringref & key) if ((c < e) && (c[0] == '}')) { key = c+1; } else { - throw IllegalArgumentException(make_string("Key '%s' is incomplete. No matching '}'", key.c_str()), VESPA_STRLOC); + throw IllegalArgumentException(make_string("Key '%s' is incomplete. No matching '}'", + vespalib::string(key).c_str()), VESPA_STRLOC); } } else { - throw IllegalArgumentException(make_string("key '%s' does not start with '{'", key.c_str()), VESPA_STRLOC); + throw IllegalArgumentException(make_string("key '%s' does not start with '{'", + vespalib::string(key).c_str()), VESPA_STRLOC); } return v; } diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index 9c64ac6a648..175fb653542 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -84,9 +84,9 @@ void reportTooShortDocId(const char * id, size_t sz) uint64_t getAsNumber(const stringref & s, const char* part) { char* errPos = NULL; - uint64_t value = strtoull(s.c_str(), &errPos, 10); + uint64_t value = strtoull(s.data(), &errPos, 10); - if (s.c_str() + s.size() != errPos) { + if (s.data() + s.size() != errPos) { reportError(s, part); } return value; @@ -96,9 +96,9 @@ void getOrderDocBits(const stringref& scheme, uint16_t & widthBits, uint16_t & divisionBits) { const char* parenPos = reinterpret_cast<const char*>( - memchr(scheme.c_str(), '(', scheme.size())); + memchr(scheme.data(), '(', scheme.size())); const char* endParenPos = reinterpret_cast<const char*>( - memchr(scheme.c_str(), ')', scheme.size())); + memchr(scheme.data(), ')', scheme.size())); if (parenPos == NULL || endParenPos == NULL || endParenPos < parenPos) { reportError(scheme); @@ -198,13 +198,13 @@ IdString::Offsets::Offsets(uint32_t maxComponents, uint32_t namespaceOffset, con { _offsets[0] = namespaceOffset; size_t index(1); - const char * s(id.c_str() + namespaceOffset); - const char * e(id.c_str() + id.size()); + const char * s(id.data() + namespaceOffset); + const char * e(id.data() + id.size()); for(s=fmemchr(s, e); (s != NULL) && (index < maxComponents); s = fmemchr(s+1, e)) { - _offsets[index++] = s - id.c_str() + 1; + _offsets[index++] = s - id.data() + 1; } _numComponents = index; for (;index < VESPA_NELEMS(_offsets); index++) { @@ -276,14 +276,14 @@ union LocationUnion { IdString::LocationType makeLocation(const stringref &s) { LocationUnion location; - fastc_md5sum(reinterpret_cast<const unsigned char*>(s.c_str()), s.size(), location._key); + fastc_md5sum(reinterpret_cast<const unsigned char*>(s.data()), s.size(), location._key); return location._location[0]; } uint64_t parseNumber(const stringref &number) { char* errPos = NULL; errno = 0; - uint64_t n = strtoul(number.c_str(), &errPos, 10); + uint64_t n = strtoul(number.data(), &errPos, 10); if (*errPos) { throw IdParseException( "'n'-value must be a 64-bit number. It was " + @@ -396,7 +396,7 @@ GroupDocIdString::locationFromGroupName(vespalib::stringref name) } OrderDocIdString::OrderDocIdString(const stringref & rawId) : - IdString(4, static_cast<const char *>(memchr(rawId.c_str(), ':', rawId.size())) - rawId.c_str() + 1, rawId), + IdString(4, static_cast<const char *>(memchr(rawId.data(), ':', rawId.size())) - rawId.data() + 1, rawId), _widthBits(0), _divisionBits(0), _ordering(getAsNumber(rawId.substr(offset(2), offset(3) - offset(2) - 1), "ordering")) diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h index 05554a68ba3..b1a14cfec94 100644 --- a/document/src/vespa/document/base/idstring.h +++ b/document/src/vespa/document/base/idstring.h @@ -24,7 +24,7 @@ public: static const vespalib::string & getTypeName(Type t); /** @throws document::IdParseException If parsing of id scheme failed. */ - static IdString::UP createIdString(const vespalib::stringref & id) { return createIdString(id.c_str(), id.size()); } + static IdString::UP createIdString(const vespalib::stringref & id) { return createIdString(id.data(), id.size()); } static IdString::UP createIdString(const char *id, size_t sz); ~IdString() {} diff --git a/document/src/vespa/document/datatype/arraydatatype.cpp b/document/src/vespa/document/datatype/arraydatatype.cpp index 5ced6053b05..b49599ac620 100644 --- a/document/src/vespa/document/datatype/arraydatatype.cpp +++ b/document/src/vespa/document/datatype/arraydatatype.cpp @@ -60,7 +60,8 @@ ArrayDataType::onBuildFieldPath(FieldPath & path, const vespalib::stringref & re if (remainFieldName[1] == '$') { path.insert(path.begin(), std::make_unique<FieldPathEntry>(getNestedType(), remainFieldName.substr(2, endPos - 2))); } else { - path.insert(path.begin(), std::make_unique<FieldPathEntry>(getNestedType(), atoi(remainFieldName.substr(1, endPos - 1).c_str()))); + // FIXME C++17 range-safe from_chars() instead of atoi() + path.insert(path.begin(), std::make_unique<FieldPathEntry>(getNestedType(), atoi(remainFieldName.substr(1, endPos - 1).data()))); } } } else { diff --git a/document/src/vespa/document/datatype/datatype.h b/document/src/vespa/document/datatype/datatype.h index 247d72db665..5857157c218 100644 --- a/document/src/vespa/document/datatype/datatype.h +++ b/document/src/vespa/document/datatype/datatype.h @@ -125,6 +125,7 @@ public: * This takes a . separated fieldname and gives you back the path of * fields you have to apply to get to your leaf. * @param remainFieldName. The remaining part of the fieldname that you want the path of. + * MUST be null-terminated. * @return pointer to field path or null if an error occured */ void buildFieldPath(FieldPath & fieldPath, const vespalib::stringref & remainFieldName) const; diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp index c7eaf42b50b..2be5acaf3db 100644 --- a/document/src/vespa/document/datatype/documenttype.cpp +++ b/document/src/vespa/document/datatype/documenttype.cpp @@ -96,7 +96,7 @@ DocumentType::addField(const Field& field) } else if (!_ownedFields.get()) { throw vespalib::IllegalStateException(make_string( "Cannot add field %s to a DocumentType that does not " - "own its fields.", field.getName().c_str()), VESPA_STRLOC); + "own its fields.", field.getName().data()), VESPA_STRLOC); } _ownedFields->addField(field); } diff --git a/document/src/vespa/document/datatype/mapdatatype.cpp b/document/src/vespa/document/datatype/mapdatatype.cpp index 5c940b1af6e..4598b96d970 100644 --- a/document/src/vespa/document/datatype/mapdatatype.cpp +++ b/document/src/vespa/document/datatype/mapdatatype.cpp @@ -73,7 +73,7 @@ MapDataType::buildFieldPathImpl(FieldPath & path, const DataType &dataType, *fv = keyValue; path.insert(path.begin(), std::make_unique<FieldPathEntry>(valueType, dataType, std::move(fv))); } - } else if (memcmp(remainFieldName.c_str(), "key", 3) == 0) { + } else if (memcmp(remainFieldName.data(), "key", 3) == 0) { size_t endPos = 3; if (remainFieldName[endPos] == '.') { endPos++; @@ -82,7 +82,7 @@ MapDataType::buildFieldPathImpl(FieldPath & path, const DataType &dataType, keyType.buildFieldPath(path, remainFieldName.substr(endPos)); path.insert(path.begin(), std::make_unique<FieldPathEntry>(dataType, keyType, valueType, true, false)); - } else if (memcmp(remainFieldName.c_str(), "value", 5) == 0) { + } else if (memcmp(remainFieldName.data(), "value", 5) == 0) { size_t endPos = 5; if (remainFieldName[endPos] == '.') { endPos++; diff --git a/document/src/vespa/document/datatype/referencedatatype.cpp b/document/src/vespa/document/datatype/referencedatatype.cpp index d02793edd3a..8f0c6a0a4c3 100644 --- a/document/src/vespa/document/datatype/referencedatatype.cpp +++ b/document/src/vespa/document/datatype/referencedatatype.cpp @@ -36,7 +36,7 @@ ReferenceDataType* ReferenceDataType::clone() const { void ReferenceDataType::onBuildFieldPath(FieldPath &, const vespalib::stringref& remainingFieldName) const { if ( ! remainingFieldName.empty() ) { throw IllegalArgumentException(make_string("Reference data type does not support further field recursion: '%s'", - remainingFieldName.c_str()), VESPA_STRLOC); + vespalib::string(remainingFieldName).c_str()), VESPA_STRLOC); } } diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp index 0abde69c2a1..9ff0b7e0b0a 100644 --- a/document/src/vespa/document/datatype/structdatatype.cpp +++ b/document/src/vespa/document/datatype/structdatatype.cpp @@ -83,7 +83,7 @@ StructDataType::addField(const Field& field) vespalib::string error = containsConflictingField(field); if (error != "") { throw IllegalArgumentException(make_string("Failed to add field '%s' to struct '%s': %s", - field.getName().c_str(), getName().c_str(), + field.getName().data(), getName().c_str(), error.c_str()), VESPA_STRLOC); } if (hasField(field.getName())) { diff --git a/document/src/vespa/document/datatype/structureddatatype.cpp b/document/src/vespa/document/datatype/structureddatatype.cpp index 41c0cd6f4e7..604b4cad045 100644 --- a/document/src/vespa/document/datatype/structureddatatype.cpp +++ b/document/src/vespa/document/datatype/structureddatatype.cpp @@ -80,7 +80,8 @@ StructuredDataType::onBuildFieldPath(FieldPath & path, const vespalib::stringref path.insert(path.begin(), std::make_unique<FieldPathEntry>(fp)); } else { throw FieldNotFoundException(currFieldName, make_string("Invalid field path '%s', no field named '%s'", - remainFieldName.c_str(), currFieldName.c_str())); + vespalib::string(remainFieldName).c_str(), + vespalib::string(currFieldName).c_str())); } } diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp index f3239553fa9..a09d5a25dd2 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp @@ -186,9 +186,6 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos, { fieldvalue::ModificationStatus retVal = ModificationStatus::NOT_MODIFIED; - LOG(spam, "iterateSubset(start=%d, end=%d, variable='%s')", - startPos, endPos, variable.c_str()); - std::vector<int> indicesToRemove; for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) { diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 47366b388a9..51ba135b826 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -32,12 +32,13 @@ void documentTypeError(const vespalib::stringref & name) __attribute__((noinline void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) __attribute__((noinline)); void documentTypeError(const vespalib::stringref & name) { - throw IllegalArgumentException(make_string("Cannot generate a document with non-document type %s.", name.c_str()), VESPA_STRLOC); + throw IllegalArgumentException(make_string("Cannot generate a document with non-document type %s.", + vespalib::string(name).c_str()), VESPA_STRLOC); } void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) { throw IllegalArgumentException(make_string("Trying to create a document with type %s that don't match the id (type %s).", - type.c_str(), docidType.c_str()), + vespalib::string(type).c_str(), vespalib::string(docidType).c_str()), VESPA_STRLOC); } diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp index 747c789c6cd..5614330a495 100644 --- a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp @@ -79,7 +79,7 @@ LiteralFieldValueB::fastCompare(const FieldValue& other) const void LiteralFieldValueB::printXml(XmlOutputStream& out) const { - out << XmlContentWrapper(_value.c_str(), _value.size()); + out << XmlContentWrapper(_value.data(), _value.size()); } void @@ -106,7 +106,7 @@ LiteralFieldValueB::getAsString() const std::pair<const char*, size_t> LiteralFieldValueB::getAsRaw() const { - return std::make_pair(_value.c_str(), _value.size()); + return std::make_pair(_value.data(), _value.size()); } void diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.h b/document/src/vespa/document/fieldvalue/literalfieldvalue.h index d72e734b6a6..2f9050eb13c 100644 --- a/document/src/vespa/document/fieldvalue/literalfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.h @@ -54,7 +54,7 @@ public: _value = _backing; _altered = true; } - size_t hash() const override final { return vespalib::hashValue(_value.c_str()); } + size_t hash() const override final { return vespalib::hashValue(_value.data(), _value.size()); } void setValue(const char* val, size_t size) { setValue(stringref(val, size)); } int compare(const FieldValue& other) const override; @@ -76,7 +76,7 @@ public: protected: void syncBacking() const __attribute__((noinline)); void sync() const { - if (__builtin_expect(_backing.c_str() != _value.c_str(), false)) { + if (__builtin_expect(_backing.data() != _value.data(), false)) { syncBacking(); } } diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp index 881cdb4a7e3..c3ef6781706 100644 --- a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp +++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp @@ -84,9 +84,10 @@ NumericFieldValue<Number>::operator=(const vespalib::stringref & value) // so detect these in front. if ((value.size() > 2) && (value[0] == '0') && ((value[1] | 0x20) == 'x')) { char* endp; - // It is safe to assume that all hex numbers can be contained within - // 64 bit unsigned value. - unsigned long long val = strtoull(value.c_str(), &endp, 16); + // It is safe to assume that all hex numbers can be contained within + // 64 bit unsigned value. + // FIXME C++17 range-safe from_chars() instead of strtoull() + unsigned long long val = strtoull(value.data(), &endp, 16); if (*endp == '\0') { // Allow numbers to be specified in range max signed to max // unsigned. These become negative numbers. diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp index 6e415471cd0..5d4abe658c1 100644 --- a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp @@ -15,13 +15,13 @@ void RawFieldValue::printXml(XmlOutputStream& out) const { out << XmlBase64Content() - << XmlContentWrapper(_value.c_str(), _value.size()); + << XmlContentWrapper(_value.data(), _value.size()); } void RawFieldValue::print(std::ostream& out, bool, const std::string&) const { - StringUtil::printAsHex(out, _value.c_str(), _value.size()); + StringUtil::printAsHex(out, _value.data(), _value.size()); } } // document diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp index 92d4e4788fb..c193b0919ea 100644 --- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp @@ -71,7 +71,7 @@ void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue & throw IllegalArgumentException( "Cannot assign value of type " + value.getDataType()->toString() + "with value : '" + value.toString() - + "' to field " + field.getName().c_str() + " of type " + + "' to field " + field.getName() + " of type " + field.getDataType().toString() + ".", VESPA_STRLOC); } setFieldValue(field, FieldValue::UP(value.clone())); diff --git a/document/src/vespa/document/select/simpleparser.cpp b/document/src/vespa/document/select/simpleparser.cpp index 462243f342c..349c1c17362 100644 --- a/document/src/vespa/document/select/simpleparser.cpp +++ b/document/src/vespa/document/select/simpleparser.cpp @@ -24,7 +24,7 @@ bool icmp(char c, char l) bool IdSpecParser::parse(const vespalib::stringref & s) { bool retval(false); - size_t pos(eatWhite(s.c_str(), s.size())); + size_t pos(eatWhite(s.data(), s.size())); if (pos+1 < s.size()) { if (icmp(s[pos], 'i') && icmp(s[pos+1],'d')) { pos += 2; @@ -77,7 +77,7 @@ bool IdSpecParser::parse(const vespalib::stringref & s) bool OperatorParser::parse(const vespalib::stringref & s) { bool retval(false); - size_t pos(eatWhite(s.c_str(), s.size())); + size_t pos(eatWhite(s.data(), s.size())); if (pos+1 < s.size()) { retval = true; if (s[pos] == '=') { @@ -122,7 +122,7 @@ bool StringParser::parse(const vespalib::stringref & s) { bool retval(false); setRemaining(s); - size_t pos(eatWhite(s.c_str(), s.size())); + size_t pos(eatWhite(s.data(), s.size())); if (pos + 1 < s.size()) { if (s[pos++] == '"') { vespalib::string str; @@ -146,7 +146,7 @@ bool StringParser::parse(const vespalib::stringref & s) bool IntegerParser::parse(const vespalib::stringref & s) { bool retval(false); - size_t pos(eatWhite(s.c_str(), s.size())); + size_t pos(eatWhite(s.data(), s.size())); if (pos < s.size()) { char * err(NULL); errno = 0; diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 08fddbaad41..a309fdd3500 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -513,7 +513,10 @@ void VespaDocumentSerializer::write(const MapValueUpdate &value) namespace { -void writeStringWithZeroTermination(nbostream & os, stringref s) +// We must ensure that string passed is always zero-terminated, so take in +// string instead of stringref. No extra allocs; function only ever called with +// string arguments. +void writeStringWithZeroTermination(nbostream & os, const vespalib::string& s) { uint32_t sz(s.size() + 1); os << sz; diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp index 051ffcf8b2a..e5a99b49a9e 100644 --- a/document/src/vespa/document/update/addvalueupdate.cpp +++ b/document/src/vespa/document/update/addvalueupdate.cpp @@ -43,7 +43,7 @@ AddValueUpdate::checkCompatibility(const Field& field) const const CollectionDataType& type(static_cast<const CollectionDataType&>(field.getDataType())); if (!type.getNestedType().isValueType(*_value)) { throw IllegalArgumentException("Cannot add value of type " + _value->getDataType()->toString() + - " to field " + field.getName().c_str() + " of container type " + + " to field " + field.getName() + " of container type " + field.getDataType().toString(), VESPA_STRLOC); } } else { diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp index 9ae7dd17a52..3af9350062e 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp +++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp @@ -35,7 +35,7 @@ ArithmeticValueUpdate::checkCompatibility(const Field& field) const if ( ! field.getDataType().inherits(NumericDataType::classId)) { throw IllegalArgumentException(vespalib::make_string( "Can not perform arithmetic update on non-numeric field '%s'.", - field.getName().c_str()), VESPA_STRLOC); + field.getName().data()), VESPA_STRLOC); } } diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp index 3fc9c8cbea5..be970b3c30a 100644 --- a/document/src/vespa/document/update/mapvalueupdate.cpp +++ b/document/src/vespa/document/update/mapvalueupdate.cpp @@ -46,7 +46,7 @@ MapValueUpdate::checkCompatibility(const Field& field) const if (_key->getClass().id() != IntFieldValue::classId) { throw IllegalArgumentException(vespalib::make_string( "Key for field '%s' is of wrong type (expected '%s', was '%s').", - field.getName().c_str(), DataType::INT->toString().c_str(), + field.getName().data(), DataType::INT->toString().c_str(), _key->getDataType()->toString().c_str()), VESPA_STRLOC); } } else if (field.getDataType().getClass().id() == WeightedSetDataType::classId) { @@ -54,7 +54,7 @@ MapValueUpdate::checkCompatibility(const Field& field) const if (!type.getNestedType().isValueType(*_key)) { throw IllegalArgumentException(vespalib::make_string( "Key for field '%s' is of wrong type (expected '%s', was '%s').", - field.getName().c_str(), DataType::INT->toString().c_str(), + field.getName().data(), DataType::INT->toString().c_str(), _key->getDataType()->toString().c_str()), VESPA_STRLOC); } } else { diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp index 28c69652a0e..fdbee3cb394 100644 --- a/document/src/vespa/document/update/removevalueupdate.cpp +++ b/document/src/vespa/document/update/removevalueupdate.cpp @@ -44,7 +44,7 @@ RemoveValueUpdate::checkCompatibility(const Field& field) const throw IllegalArgumentException( "Cannot remove value of type " + _key->getDataType()->toString() + " from field " - + field.getName().c_str() + " of container type " + + field.getName() + " of container type " + field.getDataType().toString(), VESPA_STRLOC); } } else { diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index 80548d33e72..f2ddcb38698 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -67,7 +67,7 @@ checkDimensions(const DenseTensorView &lhs, const DenseTensorView &rhs, "dense tensor %s, " "lhs dimensions = '%s', " "rhs dimensions = '%s'", - operation.c_str(), + operation.data(), dimensionsAsString(lhs.fast_type()).c_str(), dimensionsAsString(rhs.fast_type()).c_str())); } diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h index 681bdabc5eb..7e69adb5804 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h @@ -53,7 +53,7 @@ class SparseTensorUnsortedAddressBuilder ElementStringRef append(vespalib::stringref str) { - const char *cstr = str.c_str(); + const char *cstr = str.data(); uint32_t start = _elementStrings.size(); _elementStrings.insert(_elementStrings.end(), cstr, cstr + str.size() + 1); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java index 5166f53c6d2..ad6c82138e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.filter.security.athenz; import com.google.inject.Inject; import com.yahoo.jdisc.Response; @@ -9,15 +9,14 @@ import com.yahoo.jdisc.http.filter.security.cors.CorsRequestFilterBase; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; -import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; +import com.yahoo.vespa.athenz.utils.ntoken.NTokenValidator; +import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Executor; /** @@ -30,31 +29,24 @@ import java.util.concurrent.Executor; * * @author bjorncs */ -// TODO bjorncs: Move this class to vespa-athenz bundle public class AthenzPrincipalFilter extends CorsRequestFilterBase { private final NTokenValidator validator; private final String principalTokenHeader; - /** - * @param executor to preload the ZMS public keys with - */ @Inject - public AthenzPrincipalFilter(ZmsKeystore zmsKeystore, - Executor executor, - AthenzConfig athenzConfig, - CorsFilterConfig corsConfig) { - this(new NTokenValidator(zmsKeystore), executor, athenzConfig.principalHeaderName(), new HashSet<>(corsConfig.allowedUrls())); + public AthenzPrincipalFilter(AthenzPrincipalFilterConfig athenzPrincipalFilterConfig, CorsFilterConfig corsConfig) { + this(new NTokenValidator(Paths.get(athenzPrincipalFilterConfig.athenzConfFile())), + athenzPrincipalFilterConfig.principalHeaderName(), + new HashSet<>(corsConfig.allowedUrls())); } AthenzPrincipalFilter(NTokenValidator validator, - Executor executor, String principalTokenHeader, Set<String> corsAllowedUrls) { super(corsAllowedUrls); this.validator = validator; this.principalTokenHeader = principalTokenHeader; - executor.execute(validator::preloadPublicKeys); } @Override diff --git a/jdisc-security-filters/src/main/resources/configdefinitions/athenz-principal-filter.def b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-principal-filter.def new file mode 100644 index 00000000000..59e481b0d80 --- /dev/null +++ b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-principal-filter.def @@ -0,0 +1,8 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=jdisc.http.filter.security.athenz + +# Principal header name +principalHeaderName string default="Athenz-Principal-Auth" + +# Path to athenz.conf file +athenzConfFile string diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java index 301fc461b6f..be5ab9c1d77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; +package com.yahoo.jdisc.http.filter.security.athenz; -import com.yahoo.application.container.handler.Request; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ReadableContentChannel; @@ -14,8 +13,7 @@ import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.tls.KeyAlgorithm; import com.yahoo.vespa.athenz.tls.KeyUtils; import com.yahoo.vespa.athenz.tls.X509CertificateBuilder; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; +import com.yahoo.vespa.athenz.utils.ntoken.NTokenValidator; import org.junit.Before; import org.junit.Test; @@ -28,20 +26,22 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA; +import static java.util.Collections.emptyList; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.joining; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * @author bjorncs @@ -55,98 +55,115 @@ public class AthenzPrincipalFilterTest { private static final String ORIGIN = "http://localhost"; private static final Set<String> CORS_ALLOWED_URLS = singleton(ORIGIN); - private NTokenValidatorMock validator; - private ResponseHandlerMock responseHandler; + private NTokenValidator validator; @Before public void before() { - this.validator = new NTokenValidatorMock(); - this.responseHandler = new ResponseHandlerMock(); + validator = mock(NTokenValidator.class); } @Test public void valid_ntoken_is_accepted() { - Request request = defaultRequest(); - + DiscFilterRequest request = createRequestMock(); AthenzPrincipal principal = new AthenzPrincipal(IDENTITY, NTOKEN); - validator.add(NTOKEN, principal); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken()); + when(request.getClientCertificateChain()).thenReturn(emptyList()); + when(validator.validate(NTOKEN)).thenReturn(principal); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); + filter.filter(request, new ResponseHandlerMock()); - AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); - DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request); - filter.filter(filterRequest, new ResponseHandlerMock()); + verify(request).setUserPrincipal(principal); + } - assertEquals(principal, filterRequest.getUserPrincipal()); + private DiscFilterRequest createRequestMock() { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getHeader("Origin")).thenReturn(ORIGIN); + return request; } @Test public void missing_token_and_certificate_is_unauthorized() { - AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); - DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(new Request("/")); - filter.filter(filterRequest, responseHandler); + DiscFilterRequest request = createRequestMock(); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(null); + when(request.getClientCertificateChain()).thenReturn(emptyList()); + + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); + filter.filter(request, responseHandler); assertUnauthorized(responseHandler, "Unable to authenticate Athenz identity"); } @Test public void invalid_token_is_unauthorized() { - Request request = defaultRequest(); + DiscFilterRequest request = createRequestMock(); + String errorMessage = "Invalid token"; + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken()); + when(request.getClientCertificateChain()).thenReturn(emptyList()); + when(validator.validate(NTOKEN)).thenThrow(new NTokenValidator.InvalidTokenException(errorMessage)); - AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); - DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request); - filter.filter(filterRequest, responseHandler); + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); + filter.filter(request, responseHandler); - String errorMessage = "Invalid token"; assertUnauthorized(responseHandler, errorMessage); } @Test public void certificate_is_accepted() { - AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); - DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(new Request("/"), singletonList(CERTIFICATE)); - filter.filter(filterRequest, responseHandler); + DiscFilterRequest request = createRequestMock(); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(null); + when(request.getClientCertificateChain()).thenReturn(singletonList(CERTIFICATE)); + + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); + filter.filter(request, responseHandler); AthenzPrincipal expectedPrincipal = new AthenzPrincipal(IDENTITY); - assertEquals(expectedPrincipal, filterRequest.getUserPrincipal()); + verify(request).setUserPrincipal(expectedPrincipal); } @Test public void both_ntoken_and_certificate_is_accepted() { - Request request = defaultRequest(); - + DiscFilterRequest request = createRequestMock(); AthenzPrincipal principalWithToken = new AthenzPrincipal(IDENTITY, NTOKEN); - validator.add(NTOKEN, principalWithToken); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken()); + when(request.getClientCertificateChain()).thenReturn(singletonList(CERTIFICATE)); + when(validator.validate(NTOKEN)).thenReturn(principalWithToken); - AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); - DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request, singletonList(CERTIFICATE)); - filter.filter(filterRequest, responseHandler); + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); - assertEquals(principalWithToken, filterRequest.getUserPrincipal()); + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); + filter.filter(request, responseHandler); + + verify(request).setUserPrincipal(principalWithToken); } @Test public void conflicting_ntoken_and_certificate_is_unauthorized() { - Request request = defaultRequest(); - validator.add(NTOKEN, new AthenzPrincipal(IDENTITY)); - + DiscFilterRequest request = createRequestMock(); AthenzUser conflictingIdentity = AthenzUser.fromUserId("mallory"); - DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request, singletonList(createSelfSignedCertificate(conflictingIdentity))); - AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); - filter.filter(filterRequest, responseHandler); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken()); + when(request.getClientCertificateChain()) + .thenReturn(singletonList(createSelfSignedCertificate(conflictingIdentity))); + when(validator.validate(NTOKEN)).thenReturn(new AthenzPrincipal(IDENTITY)); - assertUnauthorized(responseHandler, "Identity in principal token does not match x509 CN"); - } + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); - private static Request defaultRequest() { - Request request = new Request("/"); - request.getHeaders().add("Origin", ORIGIN); - request.getHeaders().add(ATHENZ_PRINCIPAL_HEADER, NTOKEN.getRawToken()); - return request; + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS); + filter.filter(request, responseHandler); + + assertUnauthorized(responseHandler, "Identity in principal token does not match x509 CN"); } private static void assertUnauthorized(ResponseHandlerMock responseHandler, String expectedMessageSubstring) { - assertNotNull(responseHandler.response); - assertEquals(UNAUTHORIZED, responseHandler.response.getStatus()); - assertTrue(responseHandler.getResponseContent().contains(expectedMessageSubstring)); + assertThat(responseHandler.response, notNullValue()); + assertThat(responseHandler.response.getStatus(), equalTo(UNAUTHORIZED)); + assertThat(responseHandler.getResponseContent(), containsString(expectedMessageSubstring)); } private static class ResponseHandlerMock implements ResponseHandler { @@ -171,29 +188,6 @@ public class AthenzPrincipalFilterTest { } - private static class NTokenValidatorMock extends NTokenValidator { - - private final Map<NToken, AthenzPrincipal> validTokens = new HashMap<>(); - - NTokenValidatorMock() { - super((service, keyId) -> Optional.empty()); - } - - public NTokenValidatorMock add(NToken token, AthenzPrincipal principal) { - validTokens.put(token, principal); - return this; - } - - @Override - AthenzPrincipal validate(NToken token) throws InvalidTokenException { - if (!validTokens.containsKey(token)) { - throw new InvalidTokenException("Invalid token"); - } - return validTokens.get(token); - } - - } - private static X509Certificate createSelfSignedCertificate(AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 512); X500Principal x500Name = new X500Principal("CN="+ identity.getFullName()); diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 4ae3644d62c..2b31f95675b 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -34,6 +34,7 @@ import java.util.Locale; * character blocks, so if there are no definitive signs of Japanese then it is assumed that the String is Chinese. * * @author Rich Pito + * @author bjorncs */ public class SimpleDetector implements Detector { static private TextObjectFactory textObjectFactory; @@ -58,6 +59,16 @@ public class SimpleDetector implements Detector { textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); } + private final boolean enableOptimaize; + + public SimpleDetector() { + this.enableOptimaize = true; + } + + public SimpleDetector(SimpleLinguisticsConfig.Detector detector) { + this.enableOptimaize = detector.enableOptimaize(); + } + @Override public Detection detect(byte[] input, int offset, int length, Hint hint) { return new Detection(guessLanguage(input, offset, length), guessEncoding(input), false); @@ -75,11 +86,11 @@ public class SimpleDetector implements Detector { return new Detection(guessLanguage(input), Utf8.getCharset().name(), false); } - public static Language guessLanguage(byte[] buf, int offset, int length) { + public Language guessLanguage(byte[] buf, int offset, int length) { return guessLanguage(Utf8.toString(buf, offset, length)); } - public static Language guessLanguage(String input) { + public Language guessLanguage(String input) { if (input == null || input.length() == 0) { return Language.UNKNOWN; } @@ -143,7 +154,7 @@ public class SimpleDetector implements Detector { return Language.THAI; } } - if (Language.UNKNOWN.equals(soFar)){ + if (enableOptimaize && Language.UNKNOWN.equals(soFar)){ return detectLangOptimaize(input); } // got to the end, so return the current best guess diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index ad855a18088..cdfd5b4cb58 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; +import com.google.inject.Inject; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.language.Linguistics; @@ -19,15 +20,35 @@ import com.yahoo.language.process.Transformer; * Factory of pure Java linguistic processor implementations. * * @author bratseth + * @author bjorncs */ public class SimpleLinguistics implements Linguistics { // Threadsafe instances - private final static Normalizer normalizer = new SimpleNormalizer(); - private final static Transformer transformer = new SimpleTransformer(); - private final static Detector detector = new SimpleDetector(); - private final static CharacterClasses characterClasses = new CharacterClasses(); - private final static GramSplitter gramSplitter = new GramSplitter(characterClasses); + private final Normalizer normalizer; + private final Transformer transformer; + private final Detector detector; + private final CharacterClasses characterClasses; + private final GramSplitter gramSplitter; + + @Inject + public SimpleLinguistics() { + CharacterClasses characterClasses = new CharacterClasses(); + this.normalizer = new SimpleNormalizer(); + this.transformer = new SimpleTransformer(); + this.detector = new SimpleDetector(); + this.characterClasses = new CharacterClasses(); + this.gramSplitter = new GramSplitter(characterClasses); + } + + public SimpleLinguistics(SimpleLinguisticsConfig config) { + CharacterClasses characterClasses = new CharacterClasses(); + this.normalizer = new SimpleNormalizer(); + this.transformer = new SimpleTransformer(); + this.detector = new SimpleDetector(config.detector()); + this.characterClasses = new CharacterClasses(); + this.gramSplitter = new GramSplitter(characterClasses); + } @Override public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); } diff --git a/linguistics/src/main/resources/configdefinitions/simple-linguistics.def b/linguistics/src/main/resources/configdefinitions/simple-linguistics.def new file mode 100644 index 00000000000..d5e7ced7419 --- /dev/null +++ b/linguistics/src/main/resources/configdefinitions/simple-linguistics.def @@ -0,0 +1,6 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=language.simple + +# Enable Optimaize language detector +detector.enableOptimaize bool default=true + diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp index 6ed209cbe5b..0588ce5d2e7 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp @@ -6,6 +6,8 @@ #include <unistd.h> #include <sys/types.h> #include <sys/wait.h> +#include <vector> +#include <string> #include <vespa/log/log.h> LOG_SETUP(".child-handler"); @@ -15,13 +17,18 @@ ChildHandler::ChildHandler() : _childRunning(false) {} namespace { void -runSplunk(const vespalib::string &prefix, const char *a1, const char *a2 = 0) +runSplunk(const vespalib::string &prefix, std::vector<const char *> args) { - const char *argv[] = { 0, a1, a2, 0 }; vespalib::string path = prefix + "/bin/splunk"; - argv[0] = path.c_str(); - LOG(debug, "starting splunk forwarder with command: '%s' '%s' '%s'", - argv[0], argv[1], argv[2]); + args.insert(args.begin(), path.c_str()); + std::string dbg = ""; + for (const char *arg : args) { + dbg.append(" '"); + dbg.append(arg); + dbg.append("'"); + } + LOG(debug, "starting splunk forwarder with command: %s", dbg.c_str()); + args.push_back(nullptr); fflush(stdout); pid_t child = fork(); if (child == -1) { @@ -33,10 +40,10 @@ runSplunk(const vespalib::string &prefix, const char *a1, const char *a2 = 0) char *cenv = const_cast<char *>(env.c_str()); // safe cast putenv(cenv); LOG(debug, "added to environment: '%s'", cenv); - char **cargv = const_cast<char **>(argv); // safe cast - execv(argv[0], cargv); + char **cargv = const_cast<char **>(args.data()); // safe cast + execv(cargv[0], cargv); // if execv fails: - perror(argv[0]); + perror(cargv[0]); exit(1); } LOG(debug, "child running with pid %d", (int)child); @@ -69,19 +76,19 @@ void ChildHandler::startChild(const vespalib::string &prefix) { if (! _childRunning) { - runSplunk(prefix, "start", "--accept-license"); + runSplunk(prefix, {"start", "--answer-yes", "--no-prompt", "--accept-license"}); _childRunning = true; // it is possible that splunk was already running, and // then the above won't do anything, so we need to // *also* do the restart below, after a small delay. sleep(1); } - runSplunk(prefix, "restart"); + runSplunk(prefix, {"restart"}); } void ChildHandler::stopChild(const vespalib::string &prefix) { - runSplunk(prefix, "stop"); + runSplunk(prefix, {"stop"}); _childRunning = false; } diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index 04cccd59903..87c87173ec7 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -207,12 +207,12 @@ RPCSend::decode(vespalib::stringref protocolName, const vespalib::Version & vers } } else { error = Error(ErrorCode::DECODE_ERROR, - make_string("Protocol '%s' failed to decode routable.", protocolName.c_str())); + make_string("Protocol '%s' failed to decode routable.", vespalib::string(protocolName).c_str())); } } else { error = Error(ErrorCode::UNKNOWN_PROTOCOL, - make_string("Protocol '%s' is not known by %s.", protocolName.c_str(), _serverIdent.c_str())); + make_string("Protocol '%s' is not known by %s.", vespalib::string(protocolName).c_str(), _serverIdent.c_str())); } return reply; } @@ -263,7 +263,7 @@ RPCSend::invoke(FRT_RPCRequest *req) if (protocol == nullptr) { replyError(req, params->getVersion(), params->getTraceLevel(), Error(ErrorCode::UNKNOWN_PROTOCOL, make_string("Protocol '%s' is not known by %s.", - params->getProtocol().c_str(), _serverIdent.c_str()))); + vespalib::string(params->getProtocol()).c_str(), _serverIdent.c_str()))); return; } if (protocol->requireSequencing() || !_net->allowDispatchForDecode()) { @@ -284,7 +284,7 @@ RPCSend::doRequest(FRT_RPCRequest *req, const IProtocol * protocol, std::unique_ if ( ! routable ) { replyError(req, params->getVersion(), params->getTraceLevel(), Error(ErrorCode::DECODE_ERROR, - make_string("Protocol '%s' failed to decode routable.", params->getProtocol().c_str()))); + make_string("Protocol '%s' failed to decode routable.", vespalib::string(params->getProtocol()).c_str()))); return; } if (routable->isReply()) { diff --git a/messagebus/src/vespa/messagebus/routing/routeparser.cpp b/messagebus/src/vespa/messagebus/routing/routeparser.cpp index 3bc9b57d1e7..0fb90f0d585 100644 --- a/messagebus/src/vespa/messagebus/routing/routeparser.cpp +++ b/messagebus/src/vespa/messagebus/routing/routeparser.cpp @@ -34,8 +34,9 @@ RouteParser::createTcpDirective(const stringref &str) if (posS == string::npos || posS == posP + 1) { return IHopDirective::SP(); // no port } + // FIXME C++17 range-safe from_chars() instead of atoi() return IHopDirective::SP(new TcpDirective(str.substr(0, posP), - atoi(str.substr(posP + 1, posS - 1).c_str()), + atoi(str.substr(posP + 1, posS - 1).data()), str.substr(posS + 1))); } @@ -104,7 +105,7 @@ RouteParser::createHop(stringref str) return Hop().addDirective(createErrorDirective( vespalib::make_string( "Failed to completely parse '%s'.", - str.c_str()))); + vespalib::string(str).c_str()))); } else if (str[at] == '[') { ++depth; } else if (str[at] == ']') { diff --git a/metrics/src/vespa/metrics/countmetric.cpp b/metrics/src/vespa/metrics/countmetric.cpp index a19b14f9545..0c2504b2077 100644 --- a/metrics/src/vespa/metrics/countmetric.cpp +++ b/metrics/src/vespa/metrics/countmetric.cpp @@ -14,7 +14,7 @@ AbstractCountMetric::logWarning(const char* msg, const char * op) const { vespalib::asciistream ost; ost << msg << " in count metric " << getPath() << " op " << op << ". Resetting it."; - LOG(warning, "%s", ost.str().c_str()); + LOG(warning, "%s", ost.str().data()); } void diff --git a/metrics/src/vespa/metrics/valuemetric.cpp b/metrics/src/vespa/metrics/valuemetric.cpp index 04b442829ed..48baa59c7a0 100644 --- a/metrics/src/vespa/metrics/valuemetric.cpp +++ b/metrics/src/vespa/metrics/valuemetric.cpp @@ -19,7 +19,7 @@ AbstractValueMetric::logWarning(const char* msg, const char * op) const { vespalib::asciistream ost; ost << msg << " in value metric " << getPath() << " op " << op << ". Resetting it."; - LOG(warning, "%s", ost.str().c_str()); + LOG(warning, "%s", ost.str().data()); } void diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java index 71e55c36284..5bb5d35e781 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java @@ -40,22 +40,4 @@ public interface IdempotentTask<T extends TaskContext> { * @throws RuntimeException (or a subclass) if the task is unable to converge. */ boolean converge(T context); - - /** - * <p>Converge the task towards some state where it can be suspended. The - * TaskContext should provide enough to determine what kind of suspend is wanted, e.g. - * suspension of only the task, or the task and the resources/processes it manages.</p> - * - * <p>convergeSuspend() must be idempotent: it may be called any number of times, or - * interrupted at any time e.g. by `kill -9`.</p> - * - * <p>convergeSuspend() is not thread safe: The caller must ensure there is at most one - * invocation of convergeSuspend() at any given time.</p> - * - * @return false if already converged, i.e. was a no-op - * @throws RuntimeException (or a subclass) if the task is unable to suspend. - */ - default boolean convergeSuspend(T context) { - return false; - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index a7bf22591d4..ea92ca9b56f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -200,6 +200,7 @@ public class StorageMaintainer { Process duCommand = new ProcessBuilder().command(command).start(); if (!duCommand.waitFor(60, TimeUnit.SECONDS)) { duCommand.destroy(); + duCommand.waitFor(); throw new RuntimeException("Disk usage command timed out, aborting."); } String output = IOUtils.readAll(new InputStreamReader(duCommand.getInputStream())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index afc6a69796d..3b7c4857f48 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -41,8 +41,7 @@ public abstract class ApplicationMaintainer extends Maintainer { protected final void maintain() { Set<ApplicationId> applications = applicationsNeedingMaintenance(); for (ApplicationId application : applications) { - if (canDeployNow(application)) - deploy(application); + deploy(application); } } @@ -83,6 +82,7 @@ public abstract class ApplicationMaintainer extends Maintainer { // Lock is acquired with a low timeout to reduce the chance of colliding with an external deployment. try (Mutex lock = nodeRepository().lock(application, Duration.ofSeconds(1))) { if ( ! isActive(application)) return; // became inactive since deployment was requested + if ( ! canDeployNow(application)) return; // redeployment is no longer needed Optional<Deployment> deployment = deployer.deployFromLocalActive(application); if ( ! deployment.isPresent()) return; // this will be done at another config server log.log(LogLevel.DEBUG, this.getClass().getSimpleName() + " deploying " + application); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index dba50bc760b..ee5d6a04ddc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -11,7 +11,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -43,15 +42,17 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { // Returns the app that was deployed the longest time ago @Override protected Set<ApplicationId> applicationsNeedingMaintenance() { - if (waitInitially()) return new HashSet<>(); + if (waitInitially()) return Collections.emptySet(); Optional<ApplicationId> app = (nodesNeedingMaintenance().stream() .map(node -> node.allocation().get().owner()) + .distinct() .filter(this::shouldBeDeployedOnThisServer) - .min(Comparator.comparing(this::getLastDeployTime))); + .min(Comparator.comparing(this::getLastDeployTime))) + .filter(this::canDeployNow); app.ifPresent(applicationId -> log.log(LogLevel.INFO, applicationId + " will be deployed, last deploy time " + getLastDeployTime(applicationId))); - return app.map(applicationId -> new HashSet<>(Collections.singletonList(applicationId))).orElseGet(HashSet::new); + return app.map(Collections::singleton).orElseGet(Collections::emptySet); } private Instant getLastDeployTime(ApplicationId application) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index 976831d129d..961e9991d71 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -50,7 +50,7 @@ public class OperatorChangeApplicationMaintainerTest { private Fixture fixture; @Test - public void test_application_maintenance() throws InterruptedException { + public void test_application_maintenance() { ManualClock clock = new ManualClock(); Curator curator = new MockCurator(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); diff --git a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h b/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h index 6c48b02f357..508a2d04c27 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h +++ b/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h @@ -3,10 +3,9 @@ #pragma once #include "iattributevector.h" -#include <stdint.h> +#include <cstdint> -namespace search { -namespace attribute { +namespace search::attribute { /** @@ -154,7 +153,6 @@ public: } }; - typedef AttributeContent<double> FloatContent; typedef AttributeContent<const char *> ConstCharContent; typedef AttributeContent<IAttributeVector::largeint_t> IntegerContent; @@ -166,7 +164,4 @@ typedef AttributeContent<IAttributeVector::WeightedString> WeightedStringCont typedef AttributeContent<IAttributeVector::WeightedEnum> WeightedEnumContent; typedef IAttributeVector::EnumHandle EnumHandle; - -} // namespace attribute -} // namespace search - +} diff --git a/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp b/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp index c3cf0905b27..67093c686b5 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp @@ -3,8 +3,7 @@ #include <vespa/searchcommon/attribute/basictype.h> #include <vespa/vespalib/util/exceptions.h> -namespace search { -namespace attribute { +namespace search::attribute { const BasicType::TypeInfo BasicType::_typeTable[BasicType::MAX_TYPE] = { { BasicType::NONE, 0, "none" }, @@ -31,11 +30,8 @@ BasicType::asType(const vespalib::string &t) return _typeTable[i]._type; } } - throw vespalib::IllegalStateException(t + - " not recognized as " - "valid attribute data type"); + throw vespalib::IllegalStateException(t + " not recognized as valid attribute data type"); return NONE; } } -} diff --git a/searchcommon/src/vespa/searchcommon/attribute/basictype.h b/searchcommon/src/vespa/searchcommon/attribute/basictype.h index d7023aa1c59..3a2a319afd8 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/basictype.h +++ b/searchcommon/src/vespa/searchcommon/attribute/basictype.h @@ -4,8 +4,7 @@ #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace attribute { +namespace search::attribute { class BasicType { @@ -66,5 +65,3 @@ class BasicType }; } -} - diff --git a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp index 1d0dcc5328b..d5d33718183 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp @@ -3,8 +3,7 @@ #include <vespa/searchcommon/attribute/collectiontype.h> #include <vespa/vespalib/util/exceptions.h> -namespace search { -namespace attribute { +namespace search::attribute { const CollectionType::TypeInfo CollectionType::_typeTable[CollectionType::MAX_TYPE] = { { CollectionType::SINGLE, "single" }, @@ -20,11 +19,8 @@ CollectionType::asType(const vespalib::string &t) return _typeTable[i]._type; } } - throw vespalib::IllegalStateException(t + - " not recognized as valid attribute " - "collection type"); + throw vespalib::IllegalStateException(t + " not recognized as valid attribute collection type"); return SINGLE; } } -} diff --git a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h index b5b43083307..2cd46ce2bba 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h +++ b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h @@ -4,8 +4,7 @@ #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace attribute { +namespace search::attribute { class CollectionType { @@ -74,5 +73,3 @@ class CollectionType }; } -} - diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index 36dd0c94b40..221924a1689 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -20,10 +20,7 @@ Config::Config() : { } -Config::Config(BasicType bt, - CollectionType ct, - bool fastSearch_, - bool huge_) +Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) : _basicType(bt), _type(ct), _fastSearch(fastSearch_), @@ -41,6 +38,6 @@ Config::Config(BasicType bt, Config::Config(const Config &) = default; Config & Config::operator = (const Config &) = default; -Config::~Config() {} +Config::~Config() = default; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 8092d620f36..683b45b59e5 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -9,8 +9,7 @@ #include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/eval/eval/value_type.h> -namespace search { -namespace attribute { +namespace search::attribute { class Config { @@ -120,6 +119,5 @@ private: PredicateParams _predicateParams; vespalib::eval::ValueType _tensorType; }; -} // namespace attribute -} // namespace search +} diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h index 05ccbc7628e..be6b8d213d8 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h @@ -5,7 +5,6 @@ #include <vespa/searchcommon/common/range.h> #include <vespa/vespalib/stllike/string.h> - namespace search::fef { class TermFieldMatchData; } namespace search::queryeval { class SearchIterator; } namespace search { class QueryTermBase; } diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h b/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h index 2ba61512338..c283abf3ced 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h @@ -3,11 +3,8 @@ #pragma once #include "iattributevector.h" -#include <vector> -#include <memory> -namespace search { -namespace attribute { +namespace search::attribute { /** * This is an interface used to access all registered attribute vectors. @@ -53,6 +50,4 @@ public: virtual ~IAttributeContext() {} }; -} // namespace attribute -} // namespace search - +} diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h index 08bb3153838..0d3e58331c6 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h @@ -2,22 +2,21 @@ #pragma once -#include <vector> #include "collectiontype.h" #include "basictype.h" #include <vespa/searchcommon/common/iblobconverter.h> -#include <vespa/vespalib/stllike/string.h> +#include <vector> namespace search { + class IDocumentWeightAttribute; + class QueryTermSimple; +} -class IDocumentWeightAttribute; -class QueryTermSimple; - -namespace tensor { -class ITensorAttribute; +namespace search::tensor { + class ITensorAttribute; } -namespace attribute { +namespace search::attribute { class ISearchContext; class SearchContextParams; @@ -438,6 +437,4 @@ private: }; -} // namespace attribute -} // namespace search - +} diff --git a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h index 5510da7f54e..6a49bdd4d8e 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h +++ b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h @@ -4,8 +4,7 @@ #include "persistent_predicate_params.h" -namespace search { -namespace attribute { +namespace search::attribute { /* * Parameters for predicate attributes. @@ -28,5 +27,4 @@ public: } }; -} // namespace attribute -} // namespace search +} diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchcommon/src/vespa/searchcommon/attribute/status.cpp index 09b1e00a35e..d6d684a9f56 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/status.cpp @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcommon/attribute/status.h> +#include "status.h" -namespace search { -namespace attribute { +namespace search::attribute { Status::Status(const vespalib::string &) : _numDocs (0), @@ -42,8 +41,7 @@ Status::Status() vespalib::string -Status::createName(const vespalib::stringref &index, - const vespalib::stringref &attr) +Status::createName(vespalib::stringref index, vespalib::stringref attr) { vespalib::string name (index); name += ".attribute."; @@ -53,12 +51,8 @@ Status::createName(const vespalib::stringref &index, void -Status::updateStatistics(uint64_t numValues, - uint64_t numUniqueValue, - uint64_t allocated, - uint64_t used, - uint64_t dead, - uint64_t onHold) +Status::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, + uint64_t used, uint64_t dead, uint64_t onHold) { _numValues = numValues; _numUniqueValues = numUniqueValue; @@ -71,4 +65,3 @@ Status::updateStatistics(uint64_t numValues, } } -} diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchcommon/src/vespa/searchcommon/attribute/status.h index 3f6a12cb88e..dc1ccc4d5d3 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.h +++ b/searchcommon/src/vespa/searchcommon/attribute/status.h @@ -4,8 +4,7 @@ #include <vespa/vespalib/stllike/string.h> -namespace search { -namespace attribute { +namespace search::attribute { class Status { @@ -14,13 +13,8 @@ public: Status(const vespalib::string &name); Status(); - void - updateStatistics(uint64_t numValues, - uint64_t numUniqueValue, - uint64_t allocated, - uint64_t used, - uint64_t dead, - uint64_t onHold); + void updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, + uint64_t used, uint64_t dead, uint64_t onHold); uint64_t getNumDocs() const { return _numDocs; } uint64_t getNumValues() const { return _numValues; } @@ -33,32 +27,18 @@ public: uint64_t getLastSyncToken() const { return _lastSyncToken; } uint64_t getUpdateCount() const { return _updates; } uint64_t getNonIdempotentUpdateCount() const { return _nonIdempotentUpdates; } - uint32_t - getBitVectors() const - { - return _bitVectors; - } + uint32_t getBitVectors() const { return _bitVectors; } void setNumDocs(uint64_t v) { _numDocs = v; } void incNumDocs() { ++_numDocs; } void setLastSyncToken(uint64_t v) { _lastSyncToken = v; } void incUpdates(uint64_t v=1) { _updates += v; } void incNonIdempotentUpdates(uint64_t v = 1) { _nonIdempotentUpdates += v; } - void - incBitVectors() - { - ++_bitVectors; - } - - void - decBitVectors() - { - --_bitVectors; - } + void incBitVectors() { ++_bitVectors; } + void decBitVectors() { --_bitVectors; } static vespalib::string - createName(const vespalib::stringref &index, - const vespalib::stringref & attr); + createName(vespalib::stringref index, vespalib::stringref attr); private: uint64_t _numDocs; uint64_t _numValues; @@ -76,5 +56,3 @@ private: }; } -} - diff --git a/searchcommon/src/vespa/searchcommon/common/iblobconverter.h b/searchcommon/src/vespa/searchcommon/common/iblobconverter.h index f9f326b8c69..428a1d7c296 100644 --- a/searchcommon/src/vespa/searchcommon/common/iblobconverter.h +++ b/searchcommon/src/vespa/searchcommon/common/iblobconverter.h @@ -5,8 +5,7 @@ #include <vespa/vespalib/util/buffer.h> #include <memory> -namespace search { -namespace common { +namespace search::common { class BlobConverter { @@ -21,5 +20,3 @@ private: }; } -} - diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp index eafe9c28c1b..6cd9d87fa77 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp @@ -22,7 +22,7 @@ writeFields(vespalib::asciistream & os, { os << prefix << "[" << fields.size() << "]\n"; for (size_t i = 0; i < fields.size(); ++i) { - fields[i].write(os, vespalib::make_string("%s[%zu].", prefix.c_str(), i)); + fields[i].write(os, vespalib::make_string("%s[%zu].", prefix.data(), i)); } } @@ -245,7 +245,7 @@ Schema & Schema::operator=(Schema && rhs) = default; Schema::~Schema() { } bool -Schema::loadFromFile(const vespalib::stringref & fileName) +Schema::loadFromFile(const vespalib::string & fileName) { std::ifstream file(fileName.c_str()); if (!file) { @@ -284,7 +284,7 @@ Schema::loadFromFile(const vespalib::stringref & fileName) } bool -Schema::saveToFile(const vespalib::stringref & fileName) const +Schema::saveToFile(const vespalib::string & fileName) const { vespalib::asciistream os; writeToStream(os, true); diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h index 374ea840f5c..1c3ab3ccd56 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchcommon/src/vespa/searchcommon/common/schema.h @@ -179,7 +179,7 @@ public: * @return true if the schema could be loaded. **/ bool - loadFromFile(const vespalib::stringref & fileName); + loadFromFile(const vespalib::string & fileName); /** * Save this schema to the file with the given name. @@ -188,7 +188,7 @@ public: * @return true if the schema could be saved. **/ bool - saveToFile(const vespalib::stringref & fileName) const; + saveToFile(const vespalib::string & fileName) const; vespalib::string toString() const; diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 6a098667be8..d88199e6ae8 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -662,7 +662,7 @@ TEST_F("requireThatSerialNumberIsReadOnLoad", Fixture) { void crippleFusion(uint32_t fusionId) { vespalib::asciistream ost; ost << index_dir << "/index.flush." << fusionId << "/serial.dat"; - FastOS_File(ost.str().c_str()).Delete(); + FastOS_File(ost.str().data()).Delete(); } TEST_F("requireThatFailedFusionIsRetried", Fixture) { diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 045c8de9384..dfb1268aaa6 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -12,10 +12,11 @@ #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> -#include <vespa/searchcore/proton/server/i_document_db_config_owner.h> +#include <vespa/searchcore/proton/server/document_db_config_owner.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/proton_configurer.h> #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h> +#include <vespa/searchcore/proton/server/i_proton_disk_layout.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> @@ -208,13 +209,13 @@ struct ConfigFixture { struct MyProtonConfigurerOwner; -struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner +struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner { vespalib::string _name; MyProtonConfigurerOwner &_owner; MyDocumentDBConfigOwner(const vespalib::string &name, MyProtonConfigurerOwner &owner) - : IDocumentDBConfigOwner(), + : DocumentDBConfigOwner(), _name(name), _owner(owner) { @@ -224,28 +225,43 @@ struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner void reconfigure(const DocumentDBConfig::SP & config) override; }; -struct MyProtonConfigurerOwner : public IProtonConfigurerOwner +struct MyLog +{ + std::vector<vespalib::string> _log; + + MyLog() + : _log() + { + } + + void appendLog(vespalib::string logEntry) + { + _log.emplace_back(logEntry); + } +}; + +struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, + public MyLog { using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; vespalib::ThreadStackExecutor _executor; std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs; - std::vector<vespalib::string> _log; MyProtonConfigurerOwner() : IProtonConfigurerOwner(), + MyLog(), _executor(1, 128 * 1024), - _dbs(), - _log() + _dbs() { } virtual ~MyProtonConfigurerOwner() { } - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName, - document::BucketSpace bucketSpace, - const vespalib::string &configId, - const std::shared_ptr<BootstrapConfig> &bootstrapConfig, - const std::shared_ptr<DocumentDBConfig> &documentDBConfig, - InitializeThreads initializeThreads) override + virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, + document::BucketSpace bucketSpace, + const vespalib::string &configId, + const std::shared_ptr<BootstrapConfig> &bootstrapConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, + InitializeThreads initializeThreads) override { (void) bucketSpace; (void) configId; @@ -257,7 +273,7 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner std::ostringstream os; os << "add db " << docTypeName.getName() << " " << documentDBConfig->getGeneration(); _log.push_back(os.str()); - return db.get(); + return db; } virtual void removeDocumentDB(const DocTypeName &docTypeName) override { ASSERT_FALSE(_dbs.find(docTypeName) == _dbs.end()); @@ -286,17 +302,48 @@ MyDocumentDBConfigOwner::reconfigure(const DocumentDBConfig::SP & config) _owner.reconfigureDocumentDB(_name, config); } +struct MyProtonDiskLayout : public IProtonDiskLayout +{ + MyLog &_log; + + MyProtonDiskLayout(MyLog &myLog) + : _log(myLog) + { + } + void remove(const DocTypeName &docTypeName) override { + std::ostringstream os; + os << "remove dbdir " << docTypeName.getName(); + _log.appendLog(os.str()); + } + void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override { + std::ostringstream os; + os << "initial dbs "; + bool first = true; + for (const auto &docTypeName : docTypeNames) { + if (!first) { + os << ","; + } + first = false; + os << docTypeName.getName(); + } + _log.appendLog(os.str()); + } +}; + struct Fixture { MyProtonConfigurerOwner _owner; ConfigFixture _config; + std::unique_ptr<IProtonDiskLayout> _diskLayout; ProtonConfigurer _configurer; Fixture() : _owner(), _config("test"), - _configurer(_owner._executor, _owner) + _diskLayout(), + _configurer(_owner._executor, _owner, _diskLayout) { + _diskLayout = std::make_unique<MyProtonDiskLayout>(_owner); } ~Fixture() { } @@ -338,14 +385,14 @@ TEST_F("require that nothing is applied before initial config", Fixture()) TEST_F("require that initial config is applied", Fixture()) { f.applyInitialConfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that new config is blocked", Fixture()) { f.applyInitialConfig(); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that new config can be unblocked", Fixture()) @@ -353,14 +400,14 @@ TEST_F("require that new config can be unblocked", Fixture()) f.applyInitialConfig(); f.reconfigure(); f.allowReconfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); } TEST_F("require that initial config is not reapplied due to config unblock", Fixture()) { f.applyInitialConfig(); f.allowReconfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that we can add document db", Fixture()) @@ -369,7 +416,7 @@ TEST_F("require that we can add document db", Fixture()) f.allowReconfig(); f.addDocType("foobar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); } TEST_F("require that we can remove document db", Fixture()) @@ -379,7 +426,7 @@ TEST_F("require that we can remove document db", Fixture()) f.allowReconfig(); f.removeDocType("foobar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"})); } TEST_F("require that document db adds and reconfigs are intermingled", Fixture()) @@ -392,7 +439,7 @@ TEST_F("require that document db adds and reconfigs are intermingled", Fixture() f.addDocType("foobar"); f.addDocType("zbar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); } TEST_F("require that document db removes are applied at end", Fixture()) @@ -403,7 +450,7 @@ TEST_F("require that document db removes are applied at end", Fixture()) f.allowReconfig(); f.removeDocType("abar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"})); } TEST_F("require that new configs can be blocked again", Fixture()) @@ -413,7 +460,7 @@ TEST_F("require that new configs can be blocked again", Fixture()) f.allowReconfig(); f.disableReconfig(); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index 8206eba6350..0a520044985 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -68,7 +68,7 @@ public: DocsumReply::Docsum docsum; docsum.docid = 10 + i; docsum.gid = h.gid; - docsum.setData(_reply.c_str(), _reply.size()); + docsum.setData(_reply.data(), _reply.size()); retval->docsums.push_back(docsum); } return retval; diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.h b/searchcore/src/vespa/searchcore/fdispatch/search/query.h index 171a68de9ab..4d336d9843e 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/query.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.h @@ -70,6 +70,6 @@ private: const vespalib::stringref &b) { return (a.size() == b.size() && - memcmp(a.c_str(), b.c_str(), a.size()) == 0); + memcmp(a.data(), b.data(), a.size()) == 0); } }; diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp index d1731b91e6c..90c6fa2155c 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp @@ -46,7 +46,7 @@ SearchPath::parsePartList(const vespalib::stringref &partSpec, size_t numNodes) } } catch (const std::exception & e) { LOG(warning, "Failed parsing part of searchpath='%s' with error '%s'. Result might be mumbo jumbo.", - partSpec.c_str(), e.what()); + vespalib::string(partSpec).c_str(), e.what()); } } @@ -97,7 +97,8 @@ void SearchPath::parseRow(const vespalib::stringref &rowSpec) { if (!rowSpec.empty()) { - _elements.back().setRow(strtoul(rowSpec.c_str(), NULL, 0)); + // FIXME C++17 range-safe from_chars() instead of strtoul() + _elements.back().setRow(strtoul(rowSpec.data(), nullptr, 0)); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index eb6020b8d5f..442d04d30c8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -555,12 +555,12 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document } for (const auto &fupd : upd.getUpdates()) { - LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str()); + LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().data()); auto found = _attrMap.find(fupd.getField().getName()); AttributeVector * attrp = (found != _attrMap.end()) ? found->second.first : nullptr; onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (attrp == nullptr) { - LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str()); + LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().data()); continue; } // TODO: Check if we must use > due to multiple entries for same diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp index f3afab96cf5..78f73742fed 100644 --- a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp @@ -28,7 +28,7 @@ doTransactionLogReplayStart(const string &domainName, SerialNum first, SerialNum .appendKey("last").appendInt64(last) .endObject(); jstr.endObject(); - EV_STATE(eventName.c_str(), jstr.toString().c_str()); + EV_STATE(eventName.c_str(), jstr.toString().data()); } void @@ -39,7 +39,7 @@ doTransactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs, jstr.appendKey("domain").appendString(domainName); jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); jstr.endObject(); - EV_STATE(eventName.c_str(), jstr.toString().c_str()); + EV_STATE(eventName.c_str(), jstr.toString().data()); } } @@ -67,7 +67,7 @@ EventLogger::transactionLogReplayProgress(const string &domainName, float progre .appendKey("current").appendInt64(current) .endObject(); jstr.endObject(); - EV_STATE("transactionlog.replay.progress", jstr.toString().c_str()); + EV_STATE("transactionlog.replay.progress", jstr.toString().data()); } void @@ -83,7 +83,7 @@ EventLogger::flushInit(const string &name) jstr.beginObject(); jstr.appendKey("name").appendString(name); jstr.endObject(); - EV_STATE("flush.init", jstr.toString().c_str()); + EV_STATE("flush.init", jstr.toString().data()); } void @@ -105,7 +105,7 @@ EventLogger::flushStart(const string &name, int64_t beforeMemory, int64_t afterM .appendKey("current").appendInt64(current) .endObject(); jstr.endObject(); - EV_STATE("flush.start", jstr.toString().c_str()); + EV_STATE("flush.start", jstr.toString().data()); } void @@ -121,7 +121,7 @@ EventLogger::flushComplete(const string &name, int64_t elapsedTimeMs, LogUtil::logDir(jstr, outputPath, outputPathElems); } jstr.endObject(); - EV_STATE("flush.complete", jstr.toString().c_str()); + EV_STATE("flush.complete", jstr.toString().data()); } namespace { @@ -146,7 +146,7 @@ EventLogger::populateAttributeStart(const std::vector<string> &names) jstr.beginObject(); addNames(jstr, names); jstr.endObject(); - EV_STATE("populate.attribute.start", jstr.toString().c_str()); + EV_STATE("populate.attribute.start", jstr.toString().data()); } void @@ -157,7 +157,7 @@ EventLogger::populateAttributeComplete(const std::vector<string> &names, int64_t addNames(jstr, names); jstr.appendKey("documents.populated").appendInt64(documentsPopulated); jstr.endObject(); - EV_STATE("populate.attribute.complete", jstr.toString().c_str()); + EV_STATE("populate.attribute.complete", jstr.toString().data()); } void @@ -167,7 +167,7 @@ EventLogger::populateDocumentFieldStart(const string &fieldName) jstr.beginObject(); jstr.appendKey("name").appendString(fieldName); jstr.endObject(); - EV_STATE("populate.documentfield.start", jstr.toString().c_str()); + EV_STATE("populate.documentfield.start", jstr.toString().data()); } void @@ -178,7 +178,7 @@ EventLogger::populateDocumentFieldComplete(const string &fieldName, int64_t docu jstr.appendKey("name").appendString(fieldName); jstr.appendKey("documents.populated").appendInt64(documentsPopulated); jstr.endObject(); - EV_STATE("populate.documentfield.complete", jstr.toString().c_str()); + EV_STATE("populate.documentfield.complete", jstr.toString().data()); } void @@ -189,7 +189,7 @@ EventLogger::lidSpaceCompactionComplete(const string &subDbName, uint32_t lidLim jstr.appendKey("documentsubdb").appendString(subDbName); jstr.appendKey("lidlimit").appendInt64(lidLimit); jstr.endObject(); - EV_STATE("lidspace.compaction.complete", jstr.toString().c_str()); + EV_STATE("lidspace.compaction.complete", jstr.toString().data()); } @@ -201,7 +201,7 @@ EventLogger::reprocessDocumentsStart(const string &subDb, double visitCost) jstr.appendKey("documentsubdb").appendString(subDb); jstr.appendKey("visitcost").appendDouble(visitCost); jstr.endObject(); - EV_STATE("reprocess.documents.start", jstr.toString().c_str()); + EV_STATE("reprocess.documents.start", jstr.toString().data()); } @@ -214,7 +214,7 @@ EventLogger::reprocessDocumentsProgress(const string &subDb, double progress, do jstr.appendKey("progress").appendDouble(progress); jstr.appendKey("visitcost").appendDouble(visitCost); jstr.endObject(); - EV_STATE("reprocess.documents.progress", jstr.toString().c_str()); + EV_STATE("reprocess.documents.progress", jstr.toString().data()); } @@ -227,7 +227,7 @@ EventLogger::reprocessDocumentsComplete(const string &subDb, double visitCost, i jstr.appendKey("visitcost").appendDouble(visitCost); jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); jstr.endObject(); - EV_STATE("reprocess.documents.complete", jstr.toString().c_str()); + EV_STATE("reprocess.documents.complete", jstr.toString().data()); } void @@ -238,7 +238,7 @@ EventLogger::loadAttributeStart(const vespalib::string &subDbName, const vespali jstr.appendKey("documentsubdb").appendString(subDbName); jstr.appendKey("name").appendString(attrName); jstr.endObject(); - EV_STATE("load.attribute.start", jstr.toString().c_str()); + EV_STATE("load.attribute.start", jstr.toString().data()); } void @@ -251,7 +251,7 @@ EventLogger::loadAttributeComplete(const vespalib::string &subDbName, jstr.appendKey("name").appendString(attrName); jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); jstr.endObject(); - EV_STATE("load.attribute.complete", jstr.toString().c_str()); + EV_STATE("load.attribute.complete", jstr.toString().data()); } namespace { @@ -263,7 +263,7 @@ loadComponentStart(const vespalib::string &subDbName, const vespalib::string &co jstr.beginObject(); jstr.appendKey("documentsubdb").appendString(subDbName); jstr.endObject(); - EV_STATE(make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().c_str()); + EV_STATE(make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().data()); } void @@ -274,7 +274,7 @@ loadComponentComplete(const vespalib::string &subDbName, const vespalib::string jstr.appendKey("documentsubdb").appendString(subDbName); jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); jstr.endObject(); - EV_STATE(make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().c_str()); + EV_STATE(make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().data()); } } @@ -314,7 +314,7 @@ EventLogger::transactionLogPruneComplete(const string &domainName, SerialNum pru .appendKey("pruned").appendInt64(prunedSerial) .endObject(); jstr.endObject(); - EV_STATE("transactionlog.prune.complete", jstr.toString().c_str()); + EV_STATE("transactionlog.prune.complete", jstr.toString().data()); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index 85ae621db48..ea6a16e1547 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -66,10 +66,10 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type) const LiteralFieldValueB & lfv = static_cast<const LiteralFieldValueB &>(value); vespalib::stringref s = lfv.getValueRef(); - return writeStringField(s.c_str(), s.size(), type); + return writeStringField(s.data(), s.size(), type); } else { vespalib::string s = value.getAsString(); - return writeStringField(s.c_str(), s.size(), type); + return writeStringField(s.data(), s.size(), type); } } case RES_DATA: diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp index 92bb7da55e1..ecc0569cccd 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp @@ -26,7 +26,7 @@ FieldCache::FieldCache(const ResultClass &resClass, if (docType.hasField(fieldName)) { const Field &field = docType.getField(fieldName); LOG(debug, "Caching Field instance for field '%s': %s.%u", - fieldName.c_str(), field.getName().c_str(), field.getId()); + fieldName.c_str(), field.getName().data(), field.getId()); _cache.push_back(Field::CSP(new Field(field))); } else { _cache.push_back(Field::CSP()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp index 05dc75146c7..ca96033c358 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp @@ -11,8 +11,7 @@ using namespace search::query; using vespalib::make_string; using vespalib::string; -namespace proton { -namespace matching { +namespace proton::matching { AttributeLimiter::AttributeLimiter(Searchable &searchable_attributes, const IRequestContext & requestContext, @@ -35,7 +34,7 @@ AttributeLimiter::AttributeLimiter(Searchable &searchable_attributes, { } -AttributeLimiter::~AttributeLimiter() {} +AttributeLimiter::~AttributeLimiter() = default; namespace { @@ -86,5 +85,4 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool st return _blueprint->createSearch(*_match_datas.back(), strictSearch); } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h index 95f7bf4e42b..c50a6e0dcb8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h @@ -10,8 +10,7 @@ #include <vespa/searchlib/fef/matchdata.h> #include <mutex> -namespace proton { -namespace matching { +namespace proton::matching { /** * This class is responsible for creating attribute-based search @@ -51,6 +50,4 @@ private: DiversityCutoffStrategy _diversityCutoffStrategy; }; -} // namespace proton::matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h index 04440831045..c22232d47db 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h +++ b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h @@ -2,14 +2,12 @@ #pragma once -#include <vespa/searchlib/common/feature.h> #include <vespa/searchlib/queryeval/scores.h> #include <utility> #include <cstddef> #include <vector> -namespace proton { -namespace matching { +namespace proton::matching { struct IMatchLoopCommunicator { typedef search::feature_t feature_t; @@ -31,6 +29,4 @@ struct IMatchLoopCommunicator { virtual ~IMatchLoopCommunicator() {} }; -} // namespace matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp index fa41c73838b..95148ef56e8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp @@ -3,15 +3,14 @@ #include "match_loop_communicator.h" #include <vespa/vespalib/util/priority_queue.h> -namespace proton { -namespace matching { +namespace proton:: matching { MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN) : _estimate_match_frequency(threads), _selectBest(threads, topN), _rangeCover(threads) {} -MatchLoopCommunicator::~MatchLoopCommunicator() {} +MatchLoopCommunicator::~MatchLoopCommunicator() = default; void MatchLoopCommunicator::EstimateMatchFrequency::mingle() @@ -72,5 +71,4 @@ MatchLoopCommunicator::RangeCover::mingle() } } -} // namespace matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h index 3b83cb471f7..c1eec37299f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h @@ -5,8 +5,7 @@ #include "i_match_loop_communicator.h" #include <vespa/vespalib/util/rendezvous.h> -namespace proton { -namespace matching { +namespace proton::matching { class MatchLoopCommunicator : public IMatchLoopCommunicator { @@ -14,13 +13,13 @@ private: struct EstimateMatchFrequency : vespalib::Rendezvous<Matches, double> { EstimateMatchFrequency(size_t n) : vespalib::Rendezvous<Matches, double>(n) {} - virtual void mingle() override; + void mingle() override; }; struct SelectBest : vespalib::Rendezvous<std::vector<feature_t>, size_t> { size_t topN; SelectBest(size_t n, size_t topN_in) : vespalib::Rendezvous<std::vector<feature_t>, size_t>(n), topN(topN_in) {} - virtual void mingle() override; + void mingle() override; bool cmp(const uint32_t &a, const uint32_t &b) { return (in(a)[out(a)] > in(b)[out(b)]); } @@ -34,8 +33,7 @@ private: }; struct RangeCover : vespalib::Rendezvous<RangePair, RangePair> { RangeCover(size_t n) - : vespalib::Rendezvous<RangePair, RangePair>(n) {} - virtual void mingle() override; + : vespalib::Rendezvous<RangePair, RangePair>(n) {}void mingle() override; }; EstimateMatchFrequency _estimate_match_frequency; SelectBest _selectBest; @@ -45,17 +43,15 @@ public: MatchLoopCommunicator(size_t threads, size_t topN); ~MatchLoopCommunicator(); - virtual double estimate_match_frequency(const Matches &matches) override { + double estimate_match_frequency(const Matches &matches) override { return _estimate_match_frequency.rendezvous(matches); } - virtual size_t selectBest(const std::vector<feature_t> &sortedScores) override { + size_t selectBest(const std::vector<feature_t> &sortedScores) override { return _selectBest.rendezvous(sortedScores); } - virtual RangePair rangeCover(const RangePair &ranges) override { + RangePair rangeCover(const RangePair &ranges) override { return _rangeCover.rendezvous(ranges); } }; -} // namespace matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 370c4b930e1..0eb49aec754 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -10,8 +10,7 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_master"); -namespace proton { -namespace matching { +namespace proton::matching { using namespace search::fef; using search::queryeval::SearchIterator; @@ -23,15 +22,15 @@ struct TimedMatchLoopCommunicator : IMatchLoopCommunicator { IMatchLoopCommunicator &communicator; fastos::StopWatch rerank_time; TimedMatchLoopCommunicator(IMatchLoopCommunicator &com) : communicator(com) {} - virtual double estimate_match_frequency(const Matches &matches) override { + double estimate_match_frequency(const Matches &matches) override { return communicator.estimate_match_frequency(matches); } - virtual size_t selectBest(const std::vector<feature_t> &sortedScores) override { + size_t selectBest(const std::vector<feature_t> &sortedScores) override { size_t result = communicator.selectBest(sortedScores); rerank_time.start(); return result; } - virtual RangePair rangeCover(const RangePair &ranges) override { + RangePair rangeCover(const RangePair &ranges) override { RangePair result = communicator.rangeCover(ranges); rerank_time.stop(); return result; @@ -131,18 +130,15 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &matchToolsFactory, if (search.seek(docs[i])) { uint32_t docId = search.getDocId(); search.unpack(docId); - search::feature_t * f = fs.getFeaturesByIndex( - fs.addDocId(docId)); + search::feature_t * f = fs.getFeaturesByIndex(fs.addDocId(docId)); for (uint32_t j = 0; j < featureNames.size(); ++j) { f[j] = resolver.resolve(j).as_number(docId); } } else { - LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. " - "Skipping hit", docs[i]); + LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. Skipping hit", docs[i]); } } return retval; } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.h b/searchcore/src/vespa/searchcore/proton/matching/match_master.h index b9980023259..4c6463cf75d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.h @@ -8,8 +8,7 @@ namespace vespalib { class ThreadBundle; } namespace search { class FeatureSet; } -namespace proton { -namespace matching { +namespace proton::matching { class MatchToolsFactory; class MatchParams; @@ -37,6 +36,4 @@ public: static MatchingStats getStats(MatchMaster && rhs) { return std::move(rhs._stats); } }; -} // namespace proton::matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp index ddefa7c30c9..549aee20de7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp @@ -2,8 +2,7 @@ #include "match_params.h" -namespace proton { -namespace matching { +namespace proton::matching { namespace { @@ -30,8 +29,6 @@ MatchParams::MatchParams(uint32_t numDocs_in, offset(offset_in), hits(hits_in), rankDropLimit(rankDropLimit_in) -{ -} +{ } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.h b/searchcore/src/vespa/searchcore/proton/matching/match_params.h index 50787516144..c1fd9a4ef32 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_params.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.h @@ -4,8 +4,7 @@ #include <vespa/searchlib/fef/fef.h> -namespace proton { -namespace matching { +namespace proton::matching { /** * Numeric matching parameters. Some of these comes from the config, @@ -30,6 +29,4 @@ struct MatchParams { bool save_rank_scores() const { return ((heapSize + arraySize) != 0); } }; -} // namespace proton::matching -} // namespace proton - +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index d52d0fcd841..e7773c94d72 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -87,16 +87,13 @@ MatchTools::MatchTools(QueryLimiter & queryLimiter, { } -MatchTools::~MatchTools() -{ -} +MatchTools::~MatchTools() = default; void MatchTools::setup_first_phase() { setup(_rankSetup.create_first_phase_program(), - TermwiseLimit::lookup(_queryEnv.getProperties(), - _rankSetup.get_termwise_limit())); + TermwiseLimit::lookup(_queryEnv.getProperties(), _rankSetup.get_termwise_limit())); } void @@ -164,36 +161,34 @@ MatchToolsFactory(QueryLimiter & queryLimiter, double diversity_cutoff_factor = DiversityCutoffFactor::lookup(rankProperties); vespalib::string diversity_cutoff_strategy = DiversityCutoffStrategy::lookup(rankProperties); if (!limit_attribute.empty() && limit_maxhits > 0) { - _match_limiter.reset(new MatchPhaseLimiter(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext, + _match_limiter = std::make_unique<MatchPhaseLimiter>(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext, limit_attribute, limit_maxhits, !limit_ascending, limit_max_filter_coverage, samplePercentage, postFilterMultiplier, - diversity_attribute, diversity_min_groups, - diversity_cutoff_factor, - AttributeLimiter::toDiversityCutoffStrategy(diversity_cutoff_strategy))); + diversity_attribute, diversity_min_groups, diversity_cutoff_factor, + AttributeLimiter::toDiversityCutoffStrategy(diversity_cutoff_strategy)); } else if (_rankSetup.hasMatchPhaseDegradation()) { - _match_limiter.reset(new MatchPhaseLimiter(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext, + _match_limiter = std::make_unique<MatchPhaseLimiter>(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext, _rankSetup.getDegradationAttribute(), _rankSetup.getDegradationMaxHits(), !_rankSetup.isDegradationOrderAscending(), _rankSetup.getDegradationMaxFilterCoverage(), _rankSetup.getDegradationSamplePercentage(), _rankSetup.getDegradationPostFilterMultiplier(), _rankSetup.getDiversityAttribute(), _rankSetup.getDiversityMinGroups(), _rankSetup.getDiversityCutoffFactor(), - AttributeLimiter::toDiversityCutoffStrategy(_rankSetup.getDiversityCutoffStrategy()))); + AttributeLimiter::toDiversityCutoffStrategy(_rankSetup.getDiversityCutoffStrategy())); } } - if (_match_limiter.get() == nullptr) { - _match_limiter.reset(new NoMatchPhaseLimiter()); + if ( ! _match_limiter) { + _match_limiter = std::make_unique<NoMatchPhaseLimiter>(); } } -MatchToolsFactory::~MatchToolsFactory() {} +MatchToolsFactory::~MatchToolsFactory() = default; MatchTools::UP MatchToolsFactory::createMatchTools() const { assert(_valid); - return MatchTools::UP( - new MatchTools(_queryLimiter, _requestContext.getSoftDoom(), _hardDoom, _query, *_match_limiter, _queryEnv, - _mdl, _rankSetup, _featureOverrides)); + return std::make_unique<MatchTools>(_queryLimiter, _requestContext.getSoftDoom(), _hardDoom, _query, + *_match_limiter, _queryEnv, _mdl, _rankSetup, _featureOverrides); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index add4351f90b..bce13a96da7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -137,7 +137,7 @@ Matcher::Matcher(const search::index::Schema &schema, const Properties &props, c { search::features::setup_search_features(_blueprintFactory); search::fef::test::setup_fef_test_plugin(_blueprintFactory); - _rankSetup.reset(new search::fef::RankSetup(_blueprintFactory, _indexEnv)); + _rankSetup = std::make_shared<search::fef::RankSetup>(_blueprintFactory, _indexEnv); _rankSetup->configure(); // reads config values from the property map if (!_rankSetup->compile()) { throw vespalib::IllegalArgumentException("failed to compile rank setup", VESPA_STRLOC); diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index b371439dd0e..2df34312b52 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -13,6 +13,8 @@ vespa_add_library(searchcore_server STATIC disk_mem_usage_sampler.cpp disk_mem_usage_forwarder.cpp docstorevalidator.cpp + document_db_config_owner.cpp + document_db_directory_holder.cpp document_db_explorer.cpp document_db_flush_config.cpp document_db_maintenance_config.cpp @@ -73,6 +75,7 @@ vespa_add_library(searchcore_server STATIC proton_config_fetcher.cpp proton_config_snapshot.cpp proton_configurer.cpp + proton_disk_layout.cpp prune_session_cache_job.cpp pruneremoveddocumentsjob.cpp putdonecontext.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp new file mode 100644 index 00000000000..28492c5d39a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_config_owner.h" +#include "document_db_directory_holder.h" + +namespace proton { + +DocumentDBConfigOwner::DocumentDBConfigOwner() + : _holder(std::make_shared<DocumentDBDirectoryHolder>()) +{ +} + +DocumentDBConfigOwner::~DocumentDBConfigOwner() = default; + +std::shared_ptr<DocumentDBDirectoryHolder> +DocumentDBConfigOwner::getDocumentDBDirectoryHolder() +{ + return _holder; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h new file mode 100644 index 00000000000..cc19bc14fab --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_document_db_config_owner.h" + +namespace proton { + +class DocumentDBDirectoryHolder; + +/* + * Abstract class meant to be a base class for DocumentDB where a + * directory holder exists until the document db instance is + * destroyed. + */ +class DocumentDBConfigOwner : public IDocumentDBConfigOwner +{ + std::shared_ptr<DocumentDBDirectoryHolder> _holder; +public: + DocumentDBConfigOwner(); + virtual ~DocumentDBConfigOwner(); + std::shared_ptr<DocumentDBDirectoryHolder> getDocumentDBDirectoryHolder(); +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp new file mode 100644 index 00000000000..d0807a3b8a7 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_directory_holder.h" +#include <mutex> +#include <condition_variable> + +namespace proton { + +namespace { + +std::mutex mutex; +std::condition_variable cv; + +} + +DocumentDBDirectoryHolder::DocumentDBDirectoryHolder() +{ +} + +DocumentDBDirectoryHolder::~DocumentDBDirectoryHolder() +{ + std::lock_guard guard(mutex); + cv.notify_all(); +} + +void +DocumentDBDirectoryHolder::waitUntilDestroyed(const std::weak_ptr<DocumentDBDirectoryHolder> &holder) +{ + std::unique_lock guard(mutex); + cv.wait(guard, [&]() { return !holder.lock(); }); +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h new file mode 100644 index 00000000000..65df1b7bd8e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_document_db_config_owner.h" + +namespace proton { + +/* + * class holding onto a document db directory. + */ +class DocumentDBDirectoryHolder +{ +public: + DocumentDBDirectoryHolder(); + ~DocumentDBDirectoryHolder(); + static void waitUntilDestroyed(const std::weak_ptr<DocumentDBDirectoryHolder> &holder); +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index ad7a2d57c89..24d40e15677 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -90,7 +90,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, ConfigStore::UP config_store, InitializeThreads initializeThreads, const HwInfo &hwInfo) - : IDocumentDBConfigOwner(), + : DocumentDBConfigOwner(), IReplayConfig(), IFeedHandlerOwner(), IDocumentSubDBOwner(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index e25f1fe66fc..1a64a4013de 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -10,7 +10,7 @@ #include "documentsubdbcollection.h" #include "executorthreadingservice.h" #include "feedhandler.h" -#include "i_document_db_config_owner.h" +#include "document_db_config_owner.h" #include "i_document_subdb_owner.h" #include "i_feed_handler_owner.h" #include "i_lid_space_compaction_handler.h" @@ -53,7 +53,7 @@ namespace matching { class SessionManager; } * to ensure that there are never multiple writers. Unless explicitly stated, * none of the methods of this class are thread-safe. */ -class DocumentDB : public IDocumentDBConfigOwner, +class DocumentDB : public DocumentDBConfigOwner, public IReplayConfig, public IFeedHandlerOwner, public IDocumentSubDBOwner, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index a7bd61e75d4..1a1bf1ed000 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -48,7 +48,7 @@ DocumentRetriever LOG(debug, "checking document type '%s' for position fields", docTypeName.getName().c_str()); for (const document::Field * field : fields) { if (field->getDataType().getId() == positionDataTypeId) { - LOG(debug, "Field '%s' is a position field", field->getName().c_str()); + LOG(debug, "Field '%s' is a position field", field->getName().data()); const vespalib::string & zcurve_name = PositionDataType::getZCurveFieldName(field->getName()); AttributeGuard::UP attr = attr_manager.getAttribute(zcurve_name); if (attr && attr->valid()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h index 242e99fffbf..fec8430e41d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h @@ -10,7 +10,7 @@ namespace vespalib { class ThreadStackExecutorBase; } namespace proton { -class IDocumentDBConfigOwner; +class DocumentDBConfigOwner; /* * Interface class for owner of a proton configurer, with callback methods @@ -21,12 +21,12 @@ class IProtonConfigurerOwner using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; public: virtual ~IProtonConfigurerOwner() { } - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName, - document::BucketSpace bucketSpace, - const vespalib::string &configId, - const std::shared_ptr<BootstrapConfig> &bootstrapConfig, - const std::shared_ptr<DocumentDBConfig> &documentDBConfig, - InitializeThreads initializeThreads) = 0; + virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, + document::BucketSpace bucketSpace, + const vespalib::string &configId, + const std::shared_ptr<BootstrapConfig> &bootstrapConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, + InitializeThreads initializeThreads) = 0; virtual void removeDocumentDB(const DocTypeName &docTypeName) = 0; virtual void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h new file mode 100644 index 00000000000..1ee521379eb --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <set> + +namespace proton { + +class DocTypeName; + +/** + * Interface class with utility functions for handling the disk + * directory layout for proton instance. + */ +class IProtonDiskLayout +{ +public: + virtual ~IProtonDiskLayout() = default; + virtual void remove(const DocTypeName &docTypeName) = 0; + virtual void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) = 0; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp index 7d97cc4030a..d72ef274d99 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp @@ -213,7 +213,7 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList, } oss << fv[i]->getName(); } - LOG(debug, "getFlushTargets(): %zu sorted targets: [%s]", fv.size(), oss.str().c_str()); + LOG(debug, "getFlushTargets(): %zu sorted targets: [%s]", fv.size(), oss.str().data()); } return fv; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 6ffb82a2ad3..ab0abde6ccd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -9,6 +9,7 @@ #include "prepare_restart_handler.h" #include "proton.h" #include "proton_config_snapshot.h" +#include "proton_disk_layout.h" #include "resource_usage_explorer.h" #include "searchhandlerproxy.h" #include "simpleflush.h" @@ -193,7 +194,8 @@ Proton::Proton(const config::ConfigUri & configUri, // This executor can only have 1 thread as it is used for // serializing startup. _executor(1, 128 * 1024), - _protonConfigurer(_executor, *this), + _protonDiskLayout(), + _protonConfigurer(_executor, *this, _protonDiskLayout), _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _summaryExecutor(), @@ -269,7 +271,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) strategy = std::make_shared<SimpleFlush>(); break; } - vespalib::mkdir(protonConfig.basedir + "/documents", true); + _protonDiskLayout = std::make_unique<ProtonDiskLayout>(protonConfig.basedir, protonConfig.tlsspec); vespalib::chdir(protonConfig.basedir); _tls->start(); _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), @@ -352,7 +354,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) } } -IDocumentDBConfigOwner * +std::shared_ptr<DocumentDBConfigOwner> Proton::addDocumentDB(const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const vespalib::string &configId, @@ -366,21 +368,21 @@ Proton::addDocumentDB(const DocTypeName &docTypeName, if (docType != NULL) { LOG(info, "Add document database: doctypename(%s), configid(%s)", docTypeName.toString().c_str(), configId.c_str()); - return addDocumentDB(*docType, bucketSpace, bootstrapConfig, documentDBConfig, initializeThreads).get(); + return addDocumentDB(*docType, bucketSpace, bootstrapConfig, documentDBConfig, initializeThreads); } else { LOG(warning, "Did not find document type '%s' in the document manager. " "Skipping creating document database for this type", docTypeName.toString().c_str()); - return nullptr; + return std::shared_ptr<DocumentDBConfigOwner>(); } } catch (const document::DocumentTypeNotFoundException & e) { LOG(warning, "Did not find document type '%s' in the document manager. " "Skipping creating document database for this type", docTypeName.toString().c_str()); - return nullptr; + return std::shared_ptr<DocumentDBConfigOwner>(); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index dd2b42d94cf..6e07ddcecdb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -37,6 +37,7 @@ namespace proton { class DiskMemUsageSampler; class IDocumentDBReferenceRegistry; +class IProtonDiskLayout; class PrepareRestartHandler; class SummaryEngine; class DocsumBySlime; @@ -111,6 +112,7 @@ private: std::unique_ptr<vespalib::StateServer> _stateServer; std::unique_ptr<TransportServer> _fs4Server; vespalib::ThreadStackExecutor _executor; + std::unique_ptr<IProtonDiskLayout> _protonDiskLayout; ProtonConfigurer _protonConfigurer; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; @@ -130,7 +132,7 @@ private: std::mutex _nodeUpLock; std::set<BucketSpace> _nodeUp; // bucketspaces where node is up - IDocumentDBConfigOwner * + std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName & docTypeName, BucketSpace bucketSpace, const vespalib::string & configid, const BootstrapConfig::SP & bootstrapConfig, const std::shared_ptr<DocumentDBConfig> &documentDBConfig, InitializeThreads initializeThreads) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index 07ed83fd8c6..0b9293a4aab 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -4,7 +4,9 @@ #include "proton_config_snapshot.h" #include "bootstrapconfig.h" #include "i_proton_configurer_owner.h" -#include "i_document_db_config_owner.h" +#include "document_db_config_owner.h" +#include "document_db_directory_holder.h" +#include "i_proton_disk_layout.h" #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> @@ -38,7 +40,8 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name) ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, - IProtonConfigurerOwner &owner) + IProtonConfigurerOwner &owner, + const std::unique_ptr<IProtonDiskLayout> &diskLayout) : IProtonConfigurer(), _executor(executor), _owner(owner), @@ -47,7 +50,8 @@ ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, _activeConfigSnapshot(), _mutex(), _allowReconfig(false), - _componentConfig() + _componentConfig(), + _diskLayout(diskLayout) { } @@ -134,6 +138,9 @@ ProtonConfigurer::applyConfig(std::shared_ptr<ProtonConfigSnapshot> configSnapsh } const auto &bootstrapConfig = configSnapshot->getBootstrapConfig(); const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig(); + if (initialConfig) { + pruneInitialDocumentDBDirs(*configSnapshot); + } _owner.applyConfig(bootstrapConfig); for (const auto &ddbConfig : protonConfig.documentdb) { DocTypeName docTypeName(ddbConfig.inputdoctypename); @@ -162,14 +169,28 @@ ProtonConfigurer::configureDocumentDB(const ProtonConfigSnapshot &configSnapshot const auto &documentDBConfig = cfgitr->second; auto dbitr(_documentDBs.find(docTypeName)); if (dbitr == _documentDBs.end()) { - auto *newdb = _owner.addDocumentDB(docTypeName, bucketSpace, configId, bootstrapConfig, documentDBConfig, initializeThreads); - if (newdb != nullptr) { - auto insres = _documentDBs.insert(std::make_pair(docTypeName, newdb)); + auto newdb = _owner.addDocumentDB(docTypeName, bucketSpace, configId, bootstrapConfig, documentDBConfig, initializeThreads); + if (newdb) { + auto insres = _documentDBs.insert(std::make_pair(docTypeName, std::make_pair(newdb, newdb->getDocumentDBDirectoryHolder()))); assert(insres.second); } } else { - dbitr->second->reconfigure(documentDBConfig); + auto documentDB = dbitr->second.first.lock(); + assert(documentDB); + documentDB->reconfigure(documentDBConfig); + } +} + +void +ProtonConfigurer::pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot) +{ + std::set<DocTypeName> docTypeNames; + const auto &bootstrapConfig = configSnapshot.getBootstrapConfig(); + const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig(); + for (const auto &ddbConfig : protonConfig.documentdb) { + docTypeNames.emplace(ddbConfig.inputdoctypename); } + _diskLayout->initAndPruneUnused(docTypeNames); } void @@ -189,6 +210,8 @@ ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot) auto found(newDocTypes.find(dbitr->first)); if (found == newDocTypes.end()) { _owner.removeDocumentDB(dbitr->first); + DocumentDBDirectoryHolder::waitUntilDestroyed(dbitr->second.second); + _diskLayout->remove(dbitr->first); dbitr = _documentDBs.erase(dbitr); } else { ++dbitr; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index 149be3a9e62..c896f12bd4f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -12,9 +12,11 @@ namespace proton { +class DocumentDBDirectoryHolder; class IDocumentDBConfigOwner; class IProtonConfigurerOwner; class BootstrapConfig; +class IProtonDiskLayout; /* * Class to handle config changes to proton using config snapshots spanning @@ -22,7 +24,7 @@ class BootstrapConfig; */ class ProtonConfigurer : public IProtonConfigurer { - using DocumentDBs = std::map<DocTypeName, IDocumentDBConfigOwner *>; + using DocumentDBs = std::map<DocTypeName, std::pair<std::weak_ptr<IDocumentDBConfigOwner>, std::weak_ptr<DocumentDBDirectoryHolder>>>; using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; ExecutorThreadService _executor; @@ -33,6 +35,7 @@ class ProtonConfigurer : public IProtonConfigurer mutable std::mutex _mutex; bool _allowReconfig; vespalib::SimpleComponentConfigProducer _componentConfig; + const std::unique_ptr<IProtonDiskLayout> &_diskLayout; void performReconfigure(); bool skipConfig(const ProtonConfigSnapshot *configSnapshot, bool initialConfig); @@ -42,10 +45,12 @@ class ProtonConfigurer : public IProtonConfigurer const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const vespalib::string &configId, const InitializeThreads &initializeThreads); void pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot); + void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot); public: ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, - IProtonConfigurerOwner &owner); + IProtonConfigurerOwner &owner, + const std::unique_ptr<IProtonDiskLayout> &diskLayout); ~ProtonConfigurer(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp new file mode 100644 index 00000000000..1bc24d4a0e3 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "proton_disk_layout.h" +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/fastos/file.h> +#include <cassert> + +namespace proton { + +ProtonDiskLayout::ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec) + : _baseDir(baseDir), + _tlsSpec(tlsSpec) +{ + vespalib::mkdir(_baseDir + "/documents", true); +} + +ProtonDiskLayout::~ProtonDiskLayout() = default; + +void +ProtonDiskLayout::remove(const DocTypeName &docTypeName) +{ + (void) docTypeName; +} + +void +ProtonDiskLayout::initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) +{ + (void) docTypeNames; +} + +} // namespace proton + diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h new file mode 100644 index 00000000000..167a1900b99 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_proton_disk_layout.h" +#include <vespa/vespalib/stllike/string.h> + +namespace proton { + +/** + * Class with utility functions for handling the disk directory layout + * for proton instance. + */ +class ProtonDiskLayout : public IProtonDiskLayout +{ +private: + const vespalib::string _baseDir; + const vespalib::string _tlsSpec; + +public: + ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec); + ~ProtonDiskLayout() override; + void remove(const DocTypeName &docTypeName) override; + void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override; +}; + +} // namespace proton diff --git a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp b/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp index 9d824d69f93..cf1daf983b3 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp @@ -19,7 +19,7 @@ EventLogger::diskIndexLoadStart(const vespalib::string &indexDir) jstr.appendKey("input"); LogUtil::logDir(jstr, indexDir, 6); jstr.endObject(); - EV_STATE("diskindex.load.start", jstr.toString().c_str()); + EV_STATE("diskindex.load.start", jstr.toString().data()); } void @@ -32,7 +32,7 @@ EventLogger::diskIndexLoadComplete(const vespalib::string &indexDir, jstr.appendKey("input"); LogUtil::logDir(jstr, indexDir, 6); jstr.endObject(); - EV_STATE("diskindex.load.complete", jstr.toString().c_str()); + EV_STATE("diskindex.load.complete", jstr.toString().data()); } void @@ -50,7 +50,7 @@ EventLogger::diskFusionStart(const std::vector<vespalib::string> &sources, jstr.appendKey("output"); LogUtil::logDir(jstr, fusionDir, 6); jstr.endObject(); - EV_STATE("fusion.start", jstr.toString().c_str()); + EV_STATE("fusion.start", jstr.toString().data()); } void @@ -63,7 +63,7 @@ EventLogger::diskFusionComplete(const vespalib::string &fusionDir, jstr.appendKey("output"); LogUtil::logDir(jstr, fusionDir, 6); jstr.endObject(); - EV_STATE("fusion.complete", jstr.toString().c_str()); + EV_STATE("fusion.complete", jstr.toString().data()); } } diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp b/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp index 1dac8c7aafd..3180f7ed29d 100644 --- a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp +++ b/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp @@ -25,7 +25,8 @@ FactoryLoader::create(const stringref & factory) const FastOS_DynamicLibrary & lib = *_libraries.get(factory); FuncT registrationMethod = reinterpret_cast<FuncT>(lib.GetSymbol("createIndexManagerFactory")); if (registrationMethod == NULL) { - throw IllegalArgumentException(make_string("Failed locating symbol 'createIndexManagerFactory' in library '%s' for factory '%s'.", lib.GetLibName(), factory.c_str())); + throw IllegalArgumentException(make_string("Failed locating symbol 'createIndexManagerFactory' in library '%s' for factory '%s'.", + lib.GetLibName(), vespalib::string(factory).c_str())); } return IIndexManagerFactory::UP(registrationMethod()); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java index b82173eabd5..7060cfc2132 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java @@ -44,12 +44,16 @@ public class ExpressionOptimizer { return null; } - public OptimizationReport optimize(RankingExpression expression, ContextIndex arrayContext) { + public OptimizationReport optimize(RankingExpression expression, ContextIndex contextIndex) { OptimizationReport report = new OptimizationReport(); // Note: Order of optimizations matter - gbdtOptimizer.optimize(expression, arrayContext, report); - gbdtForestOptimizer.optimize(expression, arrayContext, report); + gbdtOptimizer.optimize(expression, contextIndex, report); + gbdtForestOptimizer.optimize(expression, contextIndex, report); return report; } + public OptimizationReport optimize(RankingExpression expression, AbstractArrayContext arrayContext) { + return optimize(expression, (ContextIndex)arrayContext); + } + } diff --git a/searchlib/src/tests/attribute/benchmark/attributesearcher.h b/searchlib/src/tests/attribute/benchmark/attributesearcher.h index f8cd614c48c..37f33803d27 100644 --- a/searchlib/src/tests/attribute/benchmark/attributesearcher.h +++ b/searchlib/src/tests/attribute/benchmark/attributesearcher.h @@ -126,7 +126,7 @@ AttributeFindSearcher<T>::doRun() // build simple term query vespalib::asciistream ss; ss << _values[i % _values.size()].getValue(); - this->buildTermQuery(_query, _attrPtr->getName(), ss.str().c_str()); + this->buildTermQuery(_query, _attrPtr->getName(), ss.str().data()); AttributeGuard guard(_attrPtr); std::unique_ptr<AttributeVector::SearchContext> searchContext = @@ -204,7 +204,7 @@ AttributeRangeSearcher::doRun() // build simple range term query vespalib::asciistream ss; ss << "[" << iter.a() << ";" << iter.b() << "]"; - buildTermQuery(_query, _attrPtr->getName(), ss.str().c_str()); + buildTermQuery(_query, _attrPtr->getName(), ss.str().data()); AttributeGuard guard(_attrPtr); std::unique_ptr<AttributeVector::SearchContext> searchContext = diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp index 3f49c6969a0..f025c09b6b9 100644 --- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp +++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp @@ -1,14 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("hitcollector_test"); -#include <vespa/vespalib/testkit/testapp.h> - -#include <iostream> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/queryeval/hitcollector.h> -#include <vespa/searchlib/queryeval/scores.h> + +#include <vespa/log/log.h> +LOG_SETUP("hitcollector_test"); using namespace search; using namespace search::fef; @@ -19,8 +17,8 @@ typedef std::map<uint32_t, feature_t> ScoreMap; struct BasicScorer : public HitCollector::DocumentScorer { feature_t _scoreDelta; - BasicScorer(feature_t scoreDelta) : _scoreDelta(scoreDelta) {} - virtual feature_t score(uint32_t docId) override { + explicit BasicScorer(feature_t scoreDelta) : _scoreDelta(scoreDelta) {} + feature_t score(uint32_t docId) override { return docId + _scoreDelta; } }; @@ -28,8 +26,8 @@ struct BasicScorer : public HitCollector::DocumentScorer struct PredefinedScorer : public HitCollector::DocumentScorer { ScoreMap _scores; - PredefinedScorer(const ScoreMap &scores) : _scores(scores) {} - virtual feature_t score(uint32_t docId) override { + explicit PredefinedScorer(ScoreMap scores) : _scores(std::move(scores)) {} + feature_t score(uint32_t docId) override { feature_t retval = default_rank_value; auto itr = _scores.find(docId); if (itr != _scores.end()) { @@ -41,38 +39,32 @@ struct PredefinedScorer : public HitCollector::DocumentScorer void checkResult(const ResultSet & rs, const std::vector<RankedHit> & exp) { - if (exp.size() > 0) { + if ( ! exp.empty()) { const RankedHit * rh = rs.getArray(); - ASSERT_TRUE(rh != NULL); + ASSERT_TRUE(rh != nullptr); ASSERT_EQUAL(rs.getArrayUsed(), exp.size()); for (uint32_t i = 0; i < exp.size(); ++i) { -#if 0 - std::cout << " rh[" << i << "]._docId = " << rh[i]._docId << std::endl; - std::cout << "exp[" << i << "]._docId = " << exp[i]._docId << std::endl; - std::cout << " rh[" << i << "]._rankValue = " << rh[i]._rankValue << std::endl; - std::cout << "exp[" << i << "]._rankValue = " << exp[i]._rankValue << std::endl; -#endif EXPECT_EQUAL(rh[i]._docId, exp[i]._docId); EXPECT_EQUAL(rh[i]._rankValue, exp[i]._rankValue); } } else { - ASSERT_TRUE(rs.getArray() == NULL); + ASSERT_TRUE(rs.getArray() == nullptr); } } void checkResult(ResultSet & rs, BitVector * exp) { - if (exp != NULL) { + if (exp != nullptr) { BitVector * bv = rs.getBitOverflow(); - ASSERT_TRUE(bv != NULL); + ASSERT_TRUE(bv != nullptr); bv->invalidateCachedCount(); exp->invalidateCachedCount(); LOG(info, "bv.hits: %u, exp.hits: %u", bv->countTrueBits(), exp->countTrueBits()); ASSERT_TRUE(bv->countTrueBits() == exp->countTrueBits()); EXPECT_TRUE(*bv == *exp); } else { - ASSERT_TRUE(rs.getBitOverflow() == NULL); + ASSERT_TRUE(rs.getBitOverflow() == nullptr); } } @@ -85,8 +77,8 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize) std::vector<RankedHit> expRh; std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), NULL)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } LOG(info, "testAddHit: only ranked hits"); @@ -98,14 +90,14 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize) hc.addHit(i, i + 100); // build expected result set as we go along - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), NULL)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } LOG(info, "testAddHit: both ranked hits and bit vector hits"); @@ -120,15 +112,15 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize) // build expected result set as we go along expBv->setBit(i); if (i >= (numDocs - maxHitsSize)) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, expBv.get())); } } @@ -166,14 +158,14 @@ struct Fixture { struct AscendingScoreFixture : Fixture { AscendingScoreFixture() : Fixture() {} - virtual HitRank calculateScore(uint32_t i) override { + HitRank calculateScore(uint32_t i) override { return i + 100; } }; struct DescendingScoreFixture : Fixture { DescendingScoreFixture() : Fixture() {} - virtual HitRank calculateScore(uint32_t i) override { + HitRank calculateScore(uint32_t i) override { return 100 - i; } }; @@ -197,8 +189,8 @@ TEST_F("testReRank - ascending", AscendingScoreFixture) EXPECT_EQUAL(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), f.expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, f.expBv.get())); } TEST_F("testReRank - descending", DescendingScoreFixture) @@ -216,8 +208,8 @@ TEST_F("testReRank - descending", DescendingScoreFixture) EXPECT_EQUAL(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), f.expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, f.expBv.get())); } TEST_F("testReRank - partial", AscendingScoreFixture) @@ -235,8 +227,8 @@ TEST_F("testReRank - partial", AscendingScoreFixture) EXPECT_EQUAL(expRh.size(), 10u); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), f.expBv.get())); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, f.expBv.get())); } TEST_F("require that scores for 2nd phase candidates can be retrieved", DescendingScoreFixture) @@ -251,9 +243,20 @@ TEST_F("require that scores for 2nd phase candidates can be retrieved", Descendi EXPECT_EQUAL(96, scores[4]); } +TEST_F("require that hits for 2nd phase candidates can be retrieved", DescendingScoreFixture) +{ + f.addHits(); + std::vector<HitCollector::Hit> scores = f.hc.getSortedHeapHits(); + ASSERT_EQUAL(5u, scores.size()); + EXPECT_EQUAL(100, scores[0].second); + EXPECT_EQUAL(99, scores[1].second); + EXPECT_EQUAL(98, scores[2].second); + EXPECT_EQUAL(97, scores[3].second); + EXPECT_EQUAL(96, scores[4].second); +} + TEST("require that score ranges can be read and set.") { - std::pair<Scores, Scores> ranges = - std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0)); + std::pair<Scores, Scores> ranges = std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0)); HitCollector hc(20, 10, 5); hc.setRanges(ranges); EXPECT_EQUAL(ranges.first.low, hc.getRanges().first.low); @@ -275,19 +278,19 @@ TEST("testNoHitsToReRank") { hc.addHit(i, i + 100); // build expected result set as we go along - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), NULL)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } } void testScaling(const std::vector<feature_t> &initScores, - const ScoreMap &finalScores, + ScoreMap finalScores, const std::vector<RankedHit> &expected) { HitCollector hc(5, 5, 2); @@ -297,13 +300,13 @@ void testScaling(const std::vector<feature_t> &initScores, hc.addHit(i, initScores[i]); } - PredefinedScorer scorer(finalScores); + PredefinedScorer scorer(std::move(finalScores)); // perform second phase ranking EXPECT_EQUAL(2u, hc.reRank(scorer)); // check results std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expected)); + TEST_DO(checkResult(*rs, expected)); } TEST("testScaling") { @@ -332,7 +335,7 @@ TEST("testScaling") { finalScores[3] = 300; finalScores[4] = 400; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // scale down and adjust up exp[0]._rankValue = 200; // scaled @@ -346,7 +349,7 @@ TEST("testScaling") { finalScores[3] = 500; finalScores[4] = 600; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // scale up and adjust down @@ -361,7 +364,7 @@ TEST("testScaling") { finalScores[3] = 3250; finalScores[4] = 4500; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // minimal scale (second phase range = 0 (4 - 4) -> 1) exp[0]._rankValue = 1; // scaled @@ -375,7 +378,7 @@ TEST("testScaling") { finalScores[3] = 4; finalScores[4] = 4; - testScaling(initScores, finalScores, exp); + testScaling(initScores, std::move(finalScores), exp); } { // minimal scale (first phase range = 0 (4000 - 4000) -> 1) std::vector<feature_t> is(initScores); @@ -391,7 +394,7 @@ TEST("testScaling") { finalScores[3] = 400; finalScores[4] = 500; - testScaling(is, finalScores, exp); + testScaling(is, std::move(finalScores), exp); } } @@ -410,8 +413,8 @@ TEST("testOnlyBitVector") { std::unique_ptr<ResultSet> rs = hc.getResultSet(); std::vector<RankedHit> expRh; - TEST_DO(checkResult(*rs.get(), expRh)); // no ranked hits - TEST_DO(checkResult(*rs.get(), expBv.get())); // only bit vector + TEST_DO(checkResult(*rs, expRh)); // no ranked hits + TEST_DO(checkResult(*rs, expBv.get())); // only bit vector } } @@ -433,19 +436,19 @@ TEST_F("require that result set is merged correctly with first phase ranking", f.hc.addHit(i, i + 1000); // build expected result set - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; // only the maxHitsSize best hits gets a score expRh.back()._rankValue = (i < f.numDocs - f.maxHitsSize) ? default_rank_value : i + 1000; } std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); + TEST_DO(checkResult(*rs, expRh)); } void addExpectedHitForMergeTest(const MergeResultSetFixture &f, std::vector<RankedHit> &expRh, uint32_t docId) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = docId; if (docId < f.numDocs - f.maxHitsSize) { // only the maxHitsSize best hits gets a score expRh.back()._rankValue = default_rank_value; @@ -468,7 +471,7 @@ TEST_F("require that result set is merged correctly with second phase ranking (d } EXPECT_EQUAL(f.maxHeapSize, f.hc.reRank(scorer)); std::unique_ptr<ResultSet> rs = f.hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); + TEST_DO(checkResult(*rs, expRh)); } TEST("require that hits can be added out of order") { @@ -476,7 +479,7 @@ TEST("require that hits can be added out of order") { std::vector<RankedHit> expRh; // produce expected result in normal order for (uint32_t i = 0; i < 5; ++i) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = i + 100; } @@ -485,8 +488,8 @@ TEST("require that hits can be added out of order") { hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), nullptr)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } TEST("require that hits can be added out of order when passing array limit") { @@ -495,7 +498,7 @@ TEST("require that hits can be added out of order when passing array limit") { // produce expected result in normal order const size_t numHits = 150; for (uint32_t i = 0; i < numHits; ++i) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100); } @@ -504,8 +507,8 @@ TEST("require that hits can be added out of order when passing array limit") { hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), nullptr)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } TEST("require that hits can be added out of order only after passing array limit") { @@ -514,7 +517,7 @@ TEST("require that hits can be added out of order only after passing array limit // produce expected result in normal order const size_t numHits = 150; for (uint32_t i = 0; i < numHits; ++i) { - expRh.push_back(RankedHit()); + expRh.emplace_back(); expRh.back()._docId = i; expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100); } @@ -527,8 +530,8 @@ TEST("require that hits can be added out of order only after passing array limit hc.addHit(i, i + 100); } std::unique_ptr<ResultSet> rs = hc.getResultSet(); - TEST_DO(checkResult(*rs.get(), expRh)); - TEST_DO(checkResult(*rs.get(), nullptr)); + TEST_DO(checkResult(*rs, expRh)); + TEST_DO(checkResult(*rs, nullptr)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp index 8ad4578b6c1..37a63828667 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp @@ -172,12 +172,12 @@ StackDumpIteratorTest::ShowResult(int testNo, delete item; break; } - if (strncmp(item->_indexName.c_str(), idx.c_str(), idx.size()) != 0) { + if (strncmp(item->_indexName.c_str(), idx.data(), idx.size()) != 0) { results |= ITERATOR_ERROR_WRONG_INDEX; delete item; break; } - if (strncmp(item->_term.c_str(), term.c_str(), term.size()) != 0) { + if (strncmp(item->_term.c_str(), term.data(), term.size()) != 0) { results |= ITERATOR_ERROR_WRONG_TERM; delete item; break; diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.cpp b/searchlib/src/vespa/searchlib/attribute/diversity.cpp index 14e05afa8f0..65a84baa212 100644 --- a/searchlib/src/vespa/searchlib/attribute/diversity.cpp +++ b/searchlib/src/vespa/searchlib/attribute/diversity.cpp @@ -1,11 +1,143 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "diversity.hpp" +#include "singleenumattribute.h" +#include "singlenumericattribute.h" +#include <vespa/vespalib/stllike/hash_map.h> -namespace search { -namespace attribute { -namespace diversity { +using std::make_unique; +namespace search::attribute::diversity { +template <typename T> +struct FetchNumberFast { + const T * const attr; + typedef typename T::LoadedValueType ValueType; + FetchNumberFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const T *>(&attr_in)) {} + ValueType get(uint32_t docid) const { return attr->getFast(docid); } + bool valid() const { return (attr != nullptr); } +}; + +struct FetchEnumFast { + const SingleValueEnumAttributeBase * const attr; + typedef uint32_t ValueType; + FetchEnumFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const SingleValueEnumAttributeBase *>(&attr_in)) {} + ValueType get(uint32_t docid) const { return attr->getE(docid); } + bool valid() const { return (attr != nullptr); } +}; + +struct FetchEnum { + const IAttributeVector *attr; + typedef uint32_t ValueType; + FetchEnum(const IAttributeVector & attr_in) : attr(&attr_in) {} + ValueType get(uint32_t docid) const { return attr->getEnum(docid); } +}; + +struct FetchInteger { + const IAttributeVector * attr; + typedef int64_t ValueType; + FetchInteger(const IAttributeVector & attr_in) : attr(&attr_in) {} + ValueType get(uint32_t docid) const { return attr->getInt(docid); } +}; + +struct FetchFloat { + const IAttributeVector * attr; + typedef double ValueType; + FetchFloat(const IAttributeVector & attr_in) : attr(&attr_in) {} + ValueType get(uint32_t docid) const { return attr->getFloat(docid); } +}; + +template <typename Fetcher> +class DiversityFilterT final : public DiversityFilter { +private: + size_t _total_count; + Fetcher _diversity; + size_t _max_per_group; + size_t _cutoff_max_groups; + bool _cutoff_strict; + + typedef vespalib::hash_map<typename Fetcher::ValueType, uint32_t> Diversity; + Diversity _seen; +public: + DiversityFilterT(Fetcher diversity, size_t max_per_group, size_t cutoff_max_groups, + bool cutoff_strict, size_t max_total) + : DiversityFilter(max_total), _total_count(0), _diversity(diversity), _max_per_group(max_per_group), + _cutoff_max_groups(cutoff_max_groups), _cutoff_strict(cutoff_strict), + _seen(std::min(cutoff_max_groups, 10000ul)*3) + { } + + bool accepted(uint32_t docId) override; +private: + bool add() { + ++_total_count; + return true; + } + bool conditional_add(uint32_t & group_count) { + if (group_count < _max_per_group) { + ++group_count; + add(); + return true; + } + return false; + } +}; + +template <typename Fetcher> +bool +DiversityFilterT<Fetcher>::accepted(uint32_t docId) { + if (_total_count < _max_total) { + if ((_seen.size() < _cutoff_max_groups) || _cutoff_strict) { + typename Fetcher::ValueType group = _diversity.get(docId); + if (_seen.size() < _cutoff_max_groups) { + return conditional_add(_seen[group]); + } else { + auto found = _seen.find(group); + return (found == _seen.end()) ? add() : conditional_add(found->second); + } + } else if ( !_cutoff_strict) { + return add(); + } + } + return false; } + +std::unique_ptr<DiversityFilter> +DiversityFilter::create(const IAttributeVector &diversity_attr, size_t wanted_hits, + size_t max_per_group,size_t cutoff_max_groups, bool cutoff_strict) +{ + if (diversity_attr.hasEnum()) { // must handle enum first + FetchEnumFast fastEnum(diversity_attr); + if (fastEnum.valid()) { + return make_unique<DiversityFilterT<FetchEnumFast>> (fastEnum, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } else { + return make_unique<DiversityFilterT<FetchEnum>>(FetchEnum(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } + } else if (diversity_attr.isIntegerType()) { + using FetchInt32Fast = FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > >; + using FetchInt64Fast = FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int64_t> > >; + + FetchInt32Fast fastInt32(diversity_attr); + FetchInt64Fast fastInt64(diversity_attr); + if (fastInt32.valid()) { + return make_unique<DiversityFilterT<FetchInt32Fast>>(fastInt32, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } else if (fastInt64.valid()) { + return make_unique<DiversityFilterT<FetchInt64Fast>>(fastInt64, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } else { + return make_unique<DiversityFilterT<FetchInteger>>(FetchInteger(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } + } else if (diversity_attr.isFloatingPointType()) { + using FetchFloatFast = FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<float> > >; + using FetchDoubleFast = FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<double> > >; + FetchFloatFast fastFloat(diversity_attr); + FetchDoubleFast fastDouble(diversity_attr); + if (fastFloat.valid()) { + return make_unique<DiversityFilterT<FetchFloatFast>>(fastFloat, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } else if (fastDouble.valid()) { + return make_unique<DiversityFilterT<FetchDoubleFast>>(fastDouble, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } else { + return make_unique<DiversityFilterT<FetchFloat>>(FetchFloat(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits); + } + } + return std::unique_ptr<DiversityFilter>(); } + } diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.h b/searchlib/src/vespa/searchlib/attribute/diversity.h index dff658d99d7..da5df19f9e8 100644 --- a/searchlib/src/vespa/searchlib/attribute/diversity.h +++ b/searchlib/src/vespa/searchlib/attribute/diversity.h @@ -2,9 +2,8 @@ #pragma once -#include "singleenumattribute.h" -#include "singlenumericattribute.h" -#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchlib/queryeval/idiversifier.h> /** * This file contains low-level code used to implement diversified @@ -14,9 +13,7 @@ * diversified results based on a secondary attribute. **/ -namespace search { -namespace attribute { -namespace diversity { +namespace search::attribute::diversity { template <typename ITR> class ForwardRange @@ -61,172 +58,68 @@ public: bool has_next() const { return _lower != _upper; } }; -template <typename T> -struct FetchNumberFast { - const T * const attr; - typedef typename T::LoadedValueType ValueType; - FetchNumberFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const T *>(&attr_in)) {} - ValueType get(uint32_t docid) const { return attr->getFast(docid); } - bool valid() const { return (attr != nullptr); } -}; - -struct FetchEnumFast { - const SingleValueEnumAttributeBase * const attr; - typedef uint32_t ValueType; - FetchEnumFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const SingleValueEnumAttributeBase *>(&attr_in)) {} - ValueType get(uint32_t docid) const { return attr->getE(docid); } - bool valid() const { return (attr != nullptr); } -}; - -struct FetchEnum { - const IAttributeVector &attr; - typedef uint32_t ValueType; - FetchEnum(const IAttributeVector &attr_in) : attr(attr_in) {} - ValueType get(uint32_t docid) const { return attr.getEnum(docid); } -}; - -struct FetchInteger { - const IAttributeVector &attr; - typedef int64_t ValueType; - FetchInteger(const IAttributeVector &attr_in) : attr(attr_in) {} - ValueType get(uint32_t docid) const { return attr.getInt(docid); } -}; - -struct FetchFloat { - const IAttributeVector &attr; - typedef double ValueType; - FetchFloat(const IAttributeVector &attr_in) : attr(attr_in) {} - ValueType get(uint32_t docid) const { return attr.getFloat(docid); } +class DiversityFilter : public queryeval::IDiversifier { +public: + DiversityFilter(size_t max_total) : _max_total(max_total) {} + size_t getMaxTotal() const { return _max_total; } + static std::unique_ptr<DiversityFilter> + create(const IAttributeVector &diversity_attr, size_t wanted_hits, + size_t max_per_group,size_t cutoff_max_groups, bool cutoff_strict); +protected: + size_t _max_total; }; -template <typename Fetcher, typename Result> -class DiversityFilter { +template <typename Result> +class DiversityRecorder { private: - size_t _total_count; - size_t _max_total; - const Fetcher &_diversity; - size_t _max_per_group; - size_t _cutoff_max_groups; - bool _cutoff_strict; - - typedef vespalib::hash_map<typename Fetcher::ValueType, uint32_t> Diversity; - Diversity _seen; + DiversityFilter & _filter; Result &_result; public: - DiversityFilter(const Fetcher &diversity, size_t max_per_group, - size_t cutoff_max_groups, bool cutoff_strict, - Result &result, size_t max_total) - : _total_count(0), _max_total(max_total), _diversity(diversity), _max_per_group(max_per_group), - _cutoff_max_groups(cutoff_max_groups), _cutoff_strict(cutoff_strict), - _seen(std::min(cutoff_max_groups, 10000ul)*3), _result(result) + DiversityRecorder(DiversityFilter & filter, Result &result) + : _filter(filter), _result(result) { } + template <typename Item> void push_back(Item item) { - if (_total_count < _max_total) { - if ((_seen.size() < _cutoff_max_groups) || _cutoff_strict) { - typename Fetcher::ValueType group = _diversity.get(item._key); - if (_seen.size() < _cutoff_max_groups) { - conditional_add(_seen[group], item); - } else { - auto found = _seen.find(group); - if (found == _seen.end()) { - add(item); - } else { - conditional_add(found->second, item); - } - } - } else if ( !_cutoff_strict) { - add(item); - } - } - } -private: - template <typename Item> - void add(Item item) { - ++_total_count; - _result.push_back(item); - } - template <typename Item> - void conditional_add(uint32_t & group_count, Item item) { - if (group_count < _max_per_group) { - ++group_count; - add(item); + if (_filter.accepted(item._key)) { + _result.push_back(item); } } + }; -template <typename DictRange, typename PostingStore, typename Fetcher, typename Result> -void diversify_3(const DictRange &range_in, const PostingStore &posting, size_t wanted_hits, - const Fetcher &diversity, size_t max_per_group, - size_t cutoff_max_groups, bool cutoff_strict, +template <typename DictRange, typename PostingStore, typename Result> +void diversify_2(const DictRange &range_in, const PostingStore &posting, DiversityFilter & filter, Result &result, std::vector<size_t> &fragments) { + + DiversityRecorder<Result> recorder(filter, result); DictRange range(range_in); using DataType = typename PostingStore::DataType; using KeyDataType = typename PostingStore::KeyDataType; - DiversityFilter<Fetcher, Result> filter(diversity, max_per_group, cutoff_max_groups, cutoff_strict, result, wanted_hits); - while (range.has_next() && (result.size() < wanted_hits)) { + while (range.has_next() && (result.size() < filter.getMaxTotal())) { typename DictRange::Next dict_entry(range); posting.foreach_frozen(dict_entry.get().getData(), [&](uint32_t key, const DataType &data) - { filter.push_back(KeyDataType(key, data)); }); + { recorder.push_back(KeyDataType(key, data)); }); if (fragments.back() < result.size()) { fragments.push_back(result.size()); } } } -template <typename DictRange, typename PostingStore, typename Result> -void diversify_2(const DictRange &range_in, const PostingStore &posting, size_t wanted_hits, - const IAttributeVector &diversity_attr, size_t max_per_group, - size_t cutoff_max_groups, bool cutoff_strict, - Result &result, std::vector<size_t> &fragments) -{ - if (diversity_attr.hasEnum()) { // must handle enum first - FetchEnumFast fastEnum(diversity_attr); - if (fastEnum.valid()) { - diversify_3(range_in, posting, wanted_hits, fastEnum, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } else { - diversify_3(range_in, posting, wanted_hits, FetchEnum(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } - } else if (diversity_attr.isIntegerType()) { - FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > > fastInt32(diversity_attr); - FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int64_t> > > fastInt64(diversity_attr); - if (fastInt32.valid()) { - diversify_3(range_in, posting, wanted_hits, fastInt32, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } else if (fastInt64.valid()) { - diversify_3(range_in, posting, wanted_hits, fastInt64, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } else { - diversify_3(range_in, posting, wanted_hits, FetchInteger(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } - } else if (diversity_attr.isFloatingPointType()) { - FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<float> > > fastFloat(diversity_attr); - FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<double> > > fastDouble(diversity_attr); - if (fastFloat.valid()) { - diversify_3(range_in, posting, wanted_hits, fastFloat, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } else if (fastDouble.valid()) { - diversify_3(range_in, posting, wanted_hits, fastDouble, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } else { - diversify_3(range_in, posting, wanted_hits, FetchFloat(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, result, fragments); - } - } -} - template <typename DictItr, typename PostingStore, typename Result> void diversify(bool forward, const DictItr &lower, const DictItr &upper, const PostingStore &posting, size_t wanted_hits, const IAttributeVector &diversity_attr, size_t max_per_group, size_t cutoff_max_groups, bool cutoff_strict, Result &array, std::vector<size_t> &fragments) { + auto filter = DiversityFilter::create(diversity_attr, wanted_hits, max_per_group, cutoff_max_groups, cutoff_strict); if (forward) { - diversify_2(ForwardRange<DictItr>(lower, upper), posting, wanted_hits, - diversity_attr, max_per_group, cutoff_max_groups, cutoff_strict, array, fragments); + diversify_2(ForwardRange<DictItr>(lower, upper), posting, *filter, array, fragments); } else { - diversify_2(ReverseRange<DictItr>(lower, upper), posting, wanted_hits, - diversity_attr, max_per_group, cutoff_max_groups, cutoff_strict, array, fragments); + diversify_2(ReverseRange<DictItr>(lower, upper), posting, *filter, array, fragments); } } -} // namespace search::attribute::diversity -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.hpp b/searchlib/src/vespa/searchlib/attribute/diversity.hpp index 9261bb482f9..698f482dec1 100644 --- a/searchlib/src/vespa/searchlib/attribute/diversity.hpp +++ b/searchlib/src/vespa/searchlib/attribute/diversity.hpp @@ -4,9 +4,7 @@ #include "diversity.h" -namespace search { -namespace attribute { -namespace diversity { +namespace search::attribute::diversity { template <typename ITR> ForwardRange<ITR>::ForwardRange(const ForwardRange &) = default; @@ -18,7 +16,7 @@ ForwardRange<ITR>::ForwardRange(const ITR &lower, const ITR &upper) {} template <typename ITR> -ForwardRange<ITR>::~ForwardRange() { } +ForwardRange<ITR>::~ForwardRange() = default; template <typename ITR> ReverseRange<ITR>::ReverseRange(const ReverseRange &) = default; @@ -31,8 +29,6 @@ ReverseRange<ITR>::ReverseRange(const ITR &lower, const ITR &upper) template <typename ITR> -ReverseRange<ITR>::~ReverseRange() { } +ReverseRange<ITR>::~ReverseRange() = default; } -} -} diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp index d022e806b91..39ca733eae7 100644 --- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp +++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp @@ -15,9 +15,7 @@ PostingListMerger<DataT>::PostingListMerger(uint32_t docIdLimit) } template <typename DataT> -PostingListMerger<DataT>::~PostingListMerger() -{ -} +PostingListMerger<DataT>::~PostingListMerger() = default; template <typename DataT> void diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index cfad425d7ea..cf9450cbcca 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -30,7 +30,7 @@ PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uin } template <typename DataT> -PostingListSearchContextT<DataT>::~PostingListSearchContextT() {} +PostingListSearchContextT<DataT>::~PostingListSearchContextT() = default; template <typename DataT> @@ -154,7 +154,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) { assert(_fetchPostingsDone); if (_uniqueValues == 0u) { - return SearchIterator::UP(new EmptySearch()); + return std::make_unique<EmptySearch>(); } if (_merger.hasArray() || _merger.hasBitVector()) { // synthetic results are available if (!_merger.emptyArray()) { @@ -170,7 +170,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) } } if (_merger.hasArray()) { - return SearchIterator::UP(new EmptySearch()); + return std::make_unique<EmptySearch>(); } const BitVector *bv(_merger.getBitVector()); assert(bv != nullptr); @@ -181,7 +181,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) return BitVectorIterator::create(_gbv, std::min(_gbv->size(), _docIdLimit), *matchData, strict); } if (!_pidx.valid()) { - return SearchIterator::UP(new EmptySearch()); + return std::make_unique<EmptySearch>(); } const PostingList &postingList = _postingList; if (!_frozenRoot.valid()) { diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index b6b0baea92a..90157e318e9 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -243,7 +243,7 @@ void FS4Properties::set(StringRef & e, const vespalib::stringref & s) { e.first = _backing.size(); e.second = s.size(); - _backing.append(s.c_str(), s.size()); + _backing.append(s.data(), s.size()); } void diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp index e3c5853d74e..b820a77724b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp @@ -28,8 +28,8 @@ DictionaryWordReader::~DictionaryWordReader() bool -DictionaryWordReader::open(const vespalib::stringref &dictionaryName, - const vespalib::stringref & wordMapName, +DictionaryWordReader::open(const vespalib::string & dictionaryName, + const vespalib::string & wordMapName, const TuneFileSeqRead &tuneFileRead) { _old2newwordfile.reset(new Fast_BufferedFile(new FastOS_File)); diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h index 7204ba14d25..9436c17cdbc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h @@ -106,8 +106,8 @@ public: } bool - open(const vespalib::stringref & dictionaryName, - const vespalib::stringref & wordMapName, + open(const vespalib::string & dictionaryName, + const vespalib::string & wordMapName, const TuneFileSeqRead &tuneFileRead); void diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index 99eced5f97b..b879e1f65a6 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -296,7 +296,7 @@ FileHandle::open(const vespalib::stringref &dir, index.getSchema(), index.getIndex(), tuneFileWrite, fileHeaderContext)) { LOG(error, "Could not open term writer %s for write (%s)", - dir.c_str(), getLastErrorString().c_str()); + vespalib::string(dir).c_str(), getLastErrorString().c_str()); LOG_ABORT("should not be reached"); } } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 37381cfa3f6..93f1d9c0d51 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -13,7 +13,7 @@ using vespalib::compression::decompress; using vespalib::compression::computeMaxCompressedsize; using vespalib::compression::CompressionConfig; -ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location) : +ChunkException::ChunkException(const vespalib::string & msg, const vespalib::stringref & location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) { } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h index 9f9580f1f1d..81feaf2a27f 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h @@ -12,7 +12,7 @@ namespace search { class ChunkException : public vespalib::Exception { public: - ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location); + ChunkException(const vespalib::string & msg, const vespalib::stringref & location); }; // This is an interface for implementing a chunk format diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 1f5a904a514..148d77738f9 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -934,7 +934,7 @@ LogDataStore::scanDir(const vespalib::string &dir, const vespalib::string &suffi base.c_str(), err, getLastErrorString().c_str())); } } else { - LOG(debug, "Skipping '%s' since it does not end with '%s'", file.c_str(), suffix.c_str()); + LOG(debug, "Skipping '%s' since it does not end with '%s'", file.data(), suffix.c_str()); } } } diff --git a/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp b/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp index 30c825bca65..2cfa023e7ed 100644 --- a/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp +++ b/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp @@ -12,7 +12,7 @@ SummaryException::SummaryException(const vespalib::stringref &msg, FastOS_FileInterface &file, const vespalib::stringref &location) : IoException(make_string("%s : Failing file = '%s'. Reason given by OS = '%s'", - msg.c_str(), file.GetFileName(), file.getLastErrorString().c_str()), + vespalib::string(msg).c_str(), file.GetFileName(), file.getLastErrorString().c_str()), getErrorType(file.GetLastError()), location) { } diff --git a/searchlib/src/vespa/searchlib/engine/transportserver.cpp b/searchlib/src/vespa/searchlib/engine/transportserver.cpp index 005ef473817..c94e36202a1 100644 --- a/searchlib/src/vespa/searchlib/engine/transportserver.cpp +++ b/searchlib/src/vespa/searchlib/engine/transportserver.cpp @@ -322,7 +322,7 @@ TransportServer::logPacket(const vespalib::stringref &msg, FNET_Packet *p, FNET_ } else { str = vespalib::make_string("packet { pcode=%u }", p->GetPCODE()); } - LOG(debug, "%s (chid=%u, conn=%u):\n%s", msg.c_str(), chid, conntag, str.c_str()); + LOG(debug, "%s (chid=%u, conn=%u):\n%s", msg.data(), chid, conntag, str.c_str()); } void diff --git a/searchlib/src/vespa/searchlib/expression/catserializer.cpp b/searchlib/src/vespa/searchlib/expression/catserializer.cpp index a01f68de1a4..f613bd16a54 100644 --- a/searchlib/src/vespa/searchlib/expression/catserializer.cpp +++ b/searchlib/src/vespa/searchlib/expression/catserializer.cpp @@ -17,7 +17,7 @@ using vespalib::stringref; CatSerializer & CatSerializer::put(const IFieldBase & field, const stringref & value) { (void) field; - getStream().write(value.c_str(), value.size()); + getStream().write(value.data(), value.size()); return *this; } diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index 2055cf9c18c..c7ce9528520 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -77,7 +77,8 @@ deduceResultNode(const vespalib::stringref & fieldName, const FieldValue & fv, b } else if (cInfo.inherits(MapFieldValue::classId)) { value = deduceResultNode(fieldName, *static_cast<const MapFieldValue &>(fv).createValue(), preserveAccurateTypes, nestedMultiValue); } else { - throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'", fieldName.c_str(), cInfo.name())); + throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'", + vespalib::string(fieldName).c_str(), cInfo.name())); } const Identifiable::RuntimeClass & rInfo = value->getClass(); if (rInfo.inherits(ResultNodeVector::classId)) { @@ -97,10 +98,12 @@ deduceResultNode(const vespalib::stringref & fieldName, const FieldValue & fv, b } else if (rInfo.inherits(RawResultNode::classId)) { value.reset(new RawResultNodeVector()); } else { - throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'. It nests down to %s which is not expected", fieldName.c_str(), cInfo.name(), rInfo.name())); + throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'. It nests down to %s which is not expected", + vespalib::string(fieldName).c_str(), cInfo.name(), rInfo.name())); } } else { - throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'", fieldName.c_str(), cInfo.name())); + throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'", + vespalib::string(fieldName).c_str(), cInfo.name())); } return value; } diff --git a/searchlib/src/vespa/searchlib/features/array_parser.hpp b/searchlib/src/vespa/searchlib/features/array_parser.hpp index 613fa9ec13e..92abec3aab9 100644 --- a/searchlib/src/vespa/searchlib/features/array_parser.hpp +++ b/searchlib/src/vespa/searchlib/features/array_parser.hpp @@ -52,13 +52,13 @@ ArrayParser::parsePartial(const vespalib::string &input, OutputType &output) logWarning(vespalib::make_string( "Could not parse item '%s' in query vector '%s', skipping. " "Expected ':' between dimension and component.", - item.c_str(), input.c_str())); + vespalib::string(item).c_str(), input.c_str())); return; } } catch (vespalib::IllegalArgumentException & e) { logWarning(vespalib::make_string( "Could not parse item '%s' in query vector '%s', skipping. " - "Incorrect type of operands", item.c_str(), input.c_str())); + "Incorrect type of operands", vespalib::string(item).c_str(), input.c_str())); return; } if (commaPos != vespalib::string::npos) { @@ -77,7 +77,7 @@ ArrayParser::parsePartial(const vespalib::string &input, OutputType &output) } catch (vespalib::IllegalArgumentException & e) { logWarning(vespalib::make_string( "Could not parse item[%ld] = '%s' in query vector '%s', skipping. " - "Incorrect type of operands", output.size(), is.c_str(), s.c_str())); + "Incorrect type of operands", output.size(), is.c_str(), vespalib::string(s).c_str())); return; } } diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index b400233bd99..7e913f96b7f 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -43,7 +43,7 @@ bool equals(const X & lhs, const Y & rhs) { template <> bool equals<ConstCharPtr, vespalib::stringref>(const ConstCharPtr & lhs, const vespalib::stringref & rhs) { - return strcmp(lhs, rhs.c_str()) == 0; + return strcmp(lhs, rhs.data()) == 0; } template <typename T> diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index 27e96c26538..e3cd662cdd8 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -100,7 +100,7 @@ public: EnumVector(const search::attribute::IAttributeVector * attribute) : _attribute(attribute) {} void insert(const vespalib::stringref & label, const vespalib::stringref & value) { search::attribute::EnumHandle e; - if (_attribute->findEnum(label.c_str(), e)) { + if (_attribute->findEnum(label.data(), e)) { _vector.push_back(std::make_pair(e, util::strToNum<feature_t>(value))); } } diff --git a/searchlib/src/vespa/searchlib/features/queryterm.h b/searchlib/src/vespa/searchlib/features/queryterm.h index c162b041f38..75902c33022 100644 --- a/searchlib/src/vespa/searchlib/features/queryterm.h +++ b/searchlib/src/vespa/searchlib/features/queryterm.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/fef/iqueryenvironment.h> #include <vespa/searchlib/fef/itermdata.h> -namespace search { -namespace features { +namespace search::features { /** * This class represents a query term with the relevant data. Now also @@ -58,7 +57,4 @@ public: bool lookupConnectedness = false); }; - -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index c83ec80f6b5..192fc968324 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -8,8 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalfeature"); -namespace search { -namespace features { +namespace search::features { RandomNormalExecutor::RandomNormalExecutor(uint64_t seed, double mean, double stddev) : search::fef::FeatureExecutor(), @@ -78,6 +77,4 @@ RandomNormalBlueprint::createExecutor(const search::fef::IQueryEnvironment &, ve return stash.create<RandomNormalExecutor>(seed, _mean, _stddev); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.h b/searchlib/src/vespa/searchlib/features/random_normal_feature.h index 2d2429371d9..adf3d3be63f 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.h +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/util/random_normal.h> -namespace search { -namespace features { +namespace search::features { /** @@ -60,7 +59,4 @@ public: fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; - -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp index 5f3cf7fd063..f760f52ee88 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -8,8 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalstablefeature"); -namespace search { -namespace features { +namespace search::features { RandomNormalStableExecutor::RandomNormalStableExecutor(uint64_t seed, double mean, double stddev) : search::fef::FeatureExecutor(), @@ -78,5 +77,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h index 129c929ba3d..ab9a53a2df3 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/util/random_normal.h> -namespace search { -namespace features { +namespace search::features { /** * Implements the executor for the random normal feature outputting a @@ -61,7 +60,4 @@ public: fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; - -} // namespace features -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index 937fdb23800..cdedbcadc5e 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -9,8 +9,7 @@ #include <vespa/log/log.h> LOG_SETUP(".features.randomfeature"); -namespace search { -namespace features { +namespace search::features { RandomExecutor::RandomExecutor(uint64_t seed, uint64_t matchSeed) : search::fef::FeatureExecutor(), @@ -81,5 +80,4 @@ RandomBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespa } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.h b/searchlib/src/vespa/searchlib/features/randomfeature.h index f326606009c..8ed0e403836 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.h +++ b/searchlib/src/vespa/searchlib/features/randomfeature.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/fef/featureexecutor.h> #include <vespa/searchlib/util/rand48.h> -namespace search { -namespace features { +namespace search::features { /** * Implements the executor for the random feature outputting a number in the interval [0, 1>. @@ -23,7 +22,6 @@ public: void execute(uint32_t docId) override; }; - /** * Implements the blueprint for the random feature. */ @@ -45,6 +43,4 @@ public: search::fef::FeatureExecutor &createExecutor(const search::fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index c6eeafd772f..72865d042e7 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -180,9 +180,7 @@ RankingExpressionBlueprint::RankingExpressionBlueprint(rankingexpression::Expres { } -RankingExpressionBlueprint::~RankingExpressionBlueprint() -{ -} +RankingExpressionBlueprint::~RankingExpressionBlueprint() = default; void RankingExpressionBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &, diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h index 5da6c9eae87..85514a4def7 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h @@ -8,8 +8,7 @@ #include <vespa/searchlib/features/rankingexpression/expression_replacer.h> #include <vespa/searchlib/features/rankingexpression/intrinsic_expression.h> -namespace search { -namespace features { +namespace search::features { //----------------------------------------------------------------------------- @@ -42,7 +41,4 @@ public: fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; -//----------------------------------------------------------------------------- - -} // features -} // search +} diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp index 02e44e781d4..61355581214 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp @@ -5,14 +5,14 @@ using namespace search::fef; -namespace search { -namespace features { +namespace search::features { RawScoreExecutor::RawScoreExecutor(const search::fef::IQueryEnvironment &env, uint32_t fieldId) : FeatureExecutor(), _handles(), _md(nullptr) { + _handles.reserve(env.getNumTerms()); for (uint32_t i = 0; i < env.getNumTerms(); ++i) { search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId); if (handle != search::fef::IllegalHandle) { @@ -25,8 +25,8 @@ void RawScoreExecutor::execute(uint32_t docId) { feature_t output = 0.0; - for (uint32_t i = 0; i < _handles.size(); ++i) { - const TermFieldMatchData *tfmd = _md->resolveTermField(_handles[i]); + for (auto handle : _handles) { + const TermFieldMatchData *tfmd = _md->resolveTermField(handle); if (tfmd->getDocId() == docId) { output += tfmd->getRawScore(); } @@ -43,8 +43,7 @@ RawScoreExecutor::handle_bind_match_data(const fef::MatchData &md) //----------------------------------------------------------------------------- bool -RawScoreBlueprint::setup(const IIndexEnvironment &, - const ParameterList ¶ms) +RawScoreBlueprint::setup(const IIndexEnvironment &, const ParameterList ¶ms) { _field = params[0].asField(); describeOutput("out", "accumulated raw score for the given field"); @@ -57,5 +56,4 @@ RawScoreBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::S return stash.create<RawScoreExecutor>(queryEnv, _field->id()); } -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.h b/searchlib/src/vespa/searchlib/features/raw_score_feature.h index 0eceba16ffe..db237f036c4 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.h +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.h @@ -5,8 +5,7 @@ #include <vespa/searchlib/fef/blueprint.h> #include <vespa/searchlib/fef/featureexecutor.h> -namespace search { -namespace features { +namespace search::features { class RawScoreExecutor : public fef::FeatureExecutor { @@ -40,5 +39,4 @@ public: fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp b/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp index 3199d54bf03..d67b089f5fe 100644 --- a/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp +++ b/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp @@ -24,13 +24,13 @@ WeightedSetParser::parse(const vespalib::string &input, OutputType &output) if (colonPos != vespalib::string::npos) { vespalib::string tmpKey(item.substr(0, colonPos)); vespalib::string::size_type start(tmpKey.find_first_not_of(' ')); - vespalib::stringref key(tmpKey.c_str() + start, colonPos - start); + vespalib::stringref key(tmpKey.data() + start, colonPos - start); vespalib::stringref value(item.substr(colonPos+1)); output.insert(key, value); } else { logWarning(vespalib::make_string( "Could not parse item '%s' in input string '%s', skipping. " - "Expected ':' between key and weight.", item.c_str(), input.c_str())); + "Expected ':' between key and weight.", vespalib::string(item).c_str(), input.c_str())); } if (commaPos != vespalib::string::npos) { s = s.substr(commaPos+1); diff --git a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h index a6ed0058c66..a7f268e5c6b 100644 --- a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h @@ -3,11 +3,10 @@ #pragma once #include "iindexenvironment.h" +#include "objectstore.h" #include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/searchlib/fef/objectstore.h> -namespace search { -namespace fef { +namespace search::fef { class Location; class Properties; @@ -90,5 +89,4 @@ private: ObjectStore _objectStore; }; -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/itermdata.h b/searchlib/src/vespa/searchlib/fef/itermdata.h index 94b4ed80cf0..95095dfaff0 100644 --- a/searchlib/src/vespa/searchlib/fef/itermdata.h +++ b/searchlib/src/vespa/searchlib/fef/itermdata.h @@ -6,8 +6,7 @@ #include <vespa/searchlib/query/weight.h> #include <cstddef> -namespace search { -namespace fef { +namespace search::fef { /** * Interface to static match data for a single unit (term/phrase/etc). @@ -84,6 +83,4 @@ public: void next() { ++_idx; } }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/itermfielddata.h b/searchlib/src/vespa/searchlib/fef/itermfielddata.h index a85998d8d52..bb4ab9c5285 100644 --- a/searchlib/src/vespa/searchlib/fef/itermfielddata.h +++ b/searchlib/src/vespa/searchlib/fef/itermfielddata.h @@ -4,8 +4,7 @@ #include "handle.h" -namespace search { -namespace fef { +namespace search::fef { /** * Interface to information about a single field that is being @@ -43,6 +42,4 @@ public: virtual TermFieldHandle getHandle() const = 0; }; -} // namespace fef -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h index b85ce527b8f..a62ca66486c 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.h +++ b/searchlib/src/vespa/searchlib/fef/objectstore.h @@ -3,8 +3,7 @@ #include <vespa/vespalib/stllike/hash_map.h> -namespace search { -namespace fef { +namespace search::fef { class Anything { @@ -34,4 +33,3 @@ private: }; } -} diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp index 3238243ff0b..957573d0ad7 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp @@ -256,7 +256,7 @@ FieldInverter::saveWord(const vespalib::stringref word) char * buf = &_words[0] + wordsSize; memset(buf, 0, 4); - memcpy(buf + 4, word.c_str(), len); + memcpy(buf + 4, word.data(), len); uint32_t *lastWord = reinterpret_cast<uint32_t *>(buf + 4 + (len & ~0x3)); *lastWord &= (0xffffff >> ((3 - (len & 3)) << 3)); //only on little endian machiness !! diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h index 92a69277b76..c4583d7f49f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h @@ -52,7 +52,7 @@ public: if (wordRef.valid()) { return _wordStore.getWord(wordRef); } - return _word.c_str(); + return _word.data(); } public: diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp index 527de32b956..0f8c05a44b9 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp @@ -130,7 +130,7 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, LOG(error, "Illegal field type %s for URL subfield %s, expected string", sfv->getDataType()->getName().c_str(), - subField.c_str()); + vespalib::string(subField).data()); return; } const StringFieldValue &value = static_cast<const StringFieldValue &>(*sfv); diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp index cba69ea474a..60236275f76 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp @@ -63,7 +63,7 @@ ParseItem::ParseItem(ItemType type, const vespalib::stringref & idx, const char { assert_type(type); SetType(type); - SetIndex(idx.c_str()); + SetIndex(idx.data()); SetTerm(term); } diff --git a/searchlib/src/vespa/searchlib/query/queryterm.cpp b/searchlib/src/vespa/searchlib/query/queryterm.cpp index ee2e72b41a8..5eabaf35378 100644 --- a/searchlib/src/vespa/searchlib/query/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/query/queryterm.cpp @@ -386,16 +386,16 @@ QueryTermSimple::QueryTermSimple(const string & term_, SearchTerm type) : } _valid = (numParts >= 2) && (numParts < NELEMS(parts)); if (_valid && numParts > 2) { - _rangeLimit = strtol(parts[2].c_str(), NULL, 0); + _rangeLimit = strtol(parts[2].data(), nullptr, 0); if (numParts > 3) { _valid = (numParts >= 5); if (_valid) { _diversityAttribute = parts[3]; - _maxPerGroup = strtoul(parts[4].c_str(), NULL, 0); + _maxPerGroup = strtoul(parts[4].data(), nullptr, 0); if ((_maxPerGroup > 0) && (numParts > 5)) { char *err = nullptr; - size_t cutoffGroups = strtoul(parts[5].c_str(), &err, 0); - if ((err == nullptr) || (size_t(err - parts[5].c_str()) == parts[5].size())) { + size_t cutoffGroups = strtoul(parts[5].data(), &err, 0); + if ((err == nullptr) || (size_t(err - parts[5].data()) == parts[5].size())) { _diversityCutoffGroups = cutoffGroups; } if (numParts > 6) { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index fa42cdac1c0..a208ae357fa 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -41,7 +41,7 @@ public: vespalib::stringref stack = queryStack.getStack(); LOG(error, "Unable to create query tree from stack dump. Failed at position %ld out of %ld bytes %s", queryStack.getPosition(), stack.size(), builder.error().c_str()); - LOG(error, "Raw QueryStack = %s", vespalib::HexDump(stack.c_str(), stack.size()).toString().c_str()); + LOG(error, "Raw QueryStack = %s", vespalib::HexDump(stack.data(), stack.size()).toString().c_str()); if (LOG_WOULD_LOG(debug)) { vespalib::string query = SimpleQueryStack::StackbufToString(stack); LOG(error, "Error = %s, QueryStack = %s", builder.error().c_str(), query.c_str()); diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index f9196bdaef7..e0385af99bc 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -4,8 +4,7 @@ #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/sort.h> -namespace search { -namespace queryeval { +namespace search::queryeval { void HitCollector::sortHitsByScore(size_t topn) @@ -53,16 +52,14 @@ HitCollector::HitCollector(uint32_t numDocs, _needReScore(false) { if (_maxHitsSize > 0) { - _collector.reset(new RankedHitCollector(*this)); + _collector = std::make_unique<RankedHitCollector>(*this); } else { - _collector.reset(new DocIdCollector<false>(*this)); + _collector = std::make_unique<DocIdCollector<false>>(*this); } _hits.reserve(maxHitsSize); } -HitCollector::~HitCollector() -{ -} +HitCollector::~HitCollector() = default; void HitCollector::RankedHitCollector::collect(uint32_t docId, feature_t score) @@ -113,7 +110,7 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat hc._docIdVector.push_back(hc._hits[i].first); } hc._docIdVector.push_back(docId); - newCollector.reset(new DocIdCollector<true>(hc)); + newCollector = std::make_unique<DocIdCollector<true>>(hc); } else { // start using bit vector hc._bitVector = BitVector::create(hc._numDocs); @@ -123,7 +120,7 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat hc._bitVector->setBit(hc._hits[i].first); } hc._bitVector->setBit(docId); - newCollector.reset(new BitVectorCollector<true>(hc)); + newCollector = std::make_unique<BitVectorCollector<true>>(hc); } // treat hit vector as a heap std::make_heap(hc._hits.begin(), hc._hits.end(), ScoreComparator()); @@ -168,7 +165,7 @@ HitCollector::DocIdCollector<CollectRankedHit>::collectAndChangeCollector(uint32 std::vector<uint32_t> emptyVector; emptyVector.swap(hc._docIdVector); hc._bitVector->setBit(docId); - hc._collector.reset(new BitVectorCollector<CollectRankedHit>(hc)); // note - self-destruct. + hc._collector = std::make_unique<BitVectorCollector<CollectRankedHit>>(hc); // note - self-destruct. } std::vector<feature_t> @@ -184,6 +181,20 @@ HitCollector::getSortedHeapScores() return scores; } +std::vector<HitCollector::Hit> +HitCollector::getSortedHeapHits() +{ + std::vector<Hit> scores; + size_t scoresToReturn = std::min(_hits.size(), static_cast<size_t>(_maxReRankHitsSize)); + scores.reserve(scoresToReturn); + sortHitsByScore(scoresToReturn); + for (size_t i = 0; i < scoresToReturn; ++i) { + scores.push_back(_hits[_scoreOrder[i]]); + } + return scores; +} + + size_t HitCollector::reRank(DocumentScorer &scorer) { @@ -198,24 +209,30 @@ HitCollector::reRank(DocumentScorer &scorer, size_t count) return 0; } sortHitsByScore(hitsToReRank); - _reRankedHits.reserve(_reRankedHits.size() + hitsToReRank); + std::vector<Hit> hits; + hits.reserve(hitsToReRank); for (size_t i(0); i < hitsToReRank; i++) { - _reRankedHits.push_back(_hits[_scoreOrder[i]]); + hits.push_back(_hits[_scoreOrder[i]]); } + return reRank(scorer, std::move(hits)); +} +size_t +HitCollector::reRank(DocumentScorer &scorer, std::vector<Hit> hits) { + size_t hitsToReRank = hits.size(); Scores &initScores = _ranges.first; Scores &finalScores = _ranges.second; - initScores = Scores(_reRankedHits.back().second, - _reRankedHits.front().second); + initScores = Scores(hits.back().second, hits.front().second); finalScores = Scores(std::numeric_limits<feature_t>::max(), -std::numeric_limits<feature_t>::max()); - std::sort(_reRankedHits.begin(), _reRankedHits.end()); // sort on docId - for (auto &hit : _reRankedHits) { + std::sort(hits.begin(), hits.end()); // sort on docId + for (auto &hit : hits) { hit.second = scorer.score(hit.first); finalScores.low = std::min(finalScores.low, hit.second); finalScores.high = std::max(finalScores.high, hit.second); } + _reRankedHits = std::move(hits); _hasReRanked = true; return hitsToReRank; } @@ -335,5 +352,4 @@ HitCollector::getResultSet(HitRank default_value) return rs; } -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index 1bf2bc21e95..bbd1e08c65a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -10,16 +10,14 @@ #include <vespa/vespalib/util/sort.h> #include <vespa/fastos/dynamiclibrary.h> -namespace search { - -namespace queryeval { +namespace search::queryeval { /** * This class is used to store all hits found during parallel query evaluation. **/ class HitCollector { public: - typedef std::pair<uint32_t, feature_t> Hit; + using Hit = std::pair<uint32_t, feature_t>; /** * Interface used to calculate the second phase score for the documents being re-ranked. @@ -173,6 +171,7 @@ public: * in the heap. These are the candidates for re-ranking. */ std::vector<feature_t> getSortedHeapScores(); + std::vector<Hit> getSortedHeapHits(); /** * Re-ranks the m (=maxHeapSize) best hits by invoking the score() @@ -181,6 +180,7 @@ public: **/ size_t reRank(DocumentScorer &scorer); size_t reRank(DocumentScorer &scorer, size_t count); + size_t reRank(DocumentScorer &scorer, std::vector<Hit> hits); std::pair<Scores, Scores> getRanges() const; void setRanges(const std::pair<Scores, Scores> &ranges); @@ -200,6 +200,4 @@ private: HitCollector &operator=(const HitCollector &); // Not implemented }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/idiversifier.h b/searchlib/src/vespa/searchlib/queryeval/idiversifier.h new file mode 100644 index 00000000000..e77cb959eeb --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/idiversifier.h @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace search::queryeval { + +struct IDiversifier { + virtual ~IDiversifier() {} + /** + * Will tell if this document should be kept, and update state for further filtering. + */ + virtual bool accepted(uint32_t docId) = 0; +}; +} diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp b/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp index fa8f465500e..1e0659c92e3 100644 --- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/queryeval/isourceselector.h> +#include "isourceselector.h" -namespace search { -namespace queryeval { +namespace search::queryeval { ISourceSelector::ISourceSelector(Source defaultSource) : _baseId(0), @@ -12,5 +11,3 @@ ISourceSelector::ISourceSelector(Source defaultSource) : } } - -} diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h index a3eac806558..88a3cb57a8a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h +++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h @@ -2,11 +2,9 @@ #pragma once -#include <stdint.h> #include <vespa/searchlib/attribute/singlenumericattribute.h> -namespace search { -namespace queryeval { +namespace search::queryeval { typedef uint8_t Source; @@ -91,6 +89,4 @@ private: Source _defaultSource; }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/queryeval/scores.h b/searchlib/src/vespa/searchlib/queryeval/scores.h index 4fb2de5d6fa..e8dae898909 100644 --- a/searchlib/src/vespa/searchlib/queryeval/scores.h +++ b/searchlib/src/vespa/searchlib/queryeval/scores.h @@ -4,8 +4,7 @@ #include <vespa/searchlib/common/feature.h> -namespace search { -namespace queryeval { +namespace search::queryeval { struct Scores { feature_t low; @@ -16,6 +15,4 @@ struct Scores { bool isValid() const { return low <= high; } }; -} // namespace queryeval -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp b/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp index fffcc782298..b36e9d13cdb 100644 --- a/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp +++ b/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp @@ -32,7 +32,7 @@ UcaConverter::UcaConverter(vespalib::stringref locale, vespalib::stringref stren Collator *coll(NULL); { std::lock_guard<std::mutex> guard(_GlobalDirtyICUThreadSafeLock); - coll = Collator::createInstance(icu::Locale(locale.c_str()), status); + coll = Collator::createInstance(icu::Locale(locale.data()), status); } if(U_SUCCESS(status)) { _collator.reset(coll); diff --git a/searchlib/src/vespa/searchlib/util/filekit.cpp b/searchlib/src/vespa/searchlib/util/filekit.cpp index 210a9c30134..7fc4ca8e7bb 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.cpp +++ b/searchlib/src/vespa/searchlib/util/filekit.cpp @@ -12,7 +12,7 @@ namespace search { using vespalib::getLastErrorString; bool -FileKit::createStamp(const vespalib::stringref &name) +FileKit::createStamp(const vespalib::string &name) { FastOS_File stamp; FastOS_StatInfo statInfo; @@ -40,7 +40,7 @@ FileKit::createStamp(const vespalib::stringref &name) bool -FileKit::hasStamp(const vespalib::stringref &name) +FileKit::hasStamp(const vespalib::string &name) { FastOS_StatInfo statInfo; bool statres; @@ -57,7 +57,7 @@ FileKit::hasStamp(const vespalib::stringref &name) bool -FileKit::removeStamp(const vespalib::stringref &name) +FileKit::removeStamp(const vespalib::string &name) { FastOS_StatInfo statInfo; bool deleteres; @@ -91,7 +91,7 @@ FileKit::removeStamp(const vespalib::stringref &name) fastos::TimeStamp -FileKit::getModificationTime(const vespalib::stringref &name) +FileKit::getModificationTime(const vespalib::string &name) { FastOS_StatInfo statInfo; if (FastOS_File::Stat(name.c_str(), &statInfo)) { diff --git a/searchlib/src/vespa/searchlib/util/filekit.h b/searchlib/src/vespa/searchlib/util/filekit.h index 70acf19c70c..929412ee6d8 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.h +++ b/searchlib/src/vespa/searchlib/util/filekit.h @@ -12,15 +12,15 @@ class FileKit private: static bool _syncFiles; public: - static bool createStamp(const vespalib::stringref &name); - static bool hasStamp(const vespalib::stringref &name); - static bool removeStamp(const vespalib::stringref &name); + static bool createStamp(const vespalib::string &name); + static bool hasStamp(const vespalib::string &name); + static bool removeStamp(const vespalib::string &name); /** * Returns the modification time of the given file/directory, * or time stamp 0 if stating of file/directory fails. **/ - static fastos::TimeStamp getModificationTime(const vespalib::stringref &name); + static fastos::TimeStamp getModificationTime(const vespalib::string &name); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index c7eb63a4480..73c62db864c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -159,7 +159,7 @@ public: { if (item->_si != NULL) { *len = item->_si->getIndexName().size(); - return item->_si->getIndexName().c_str(); + return item->_si->getIndexName().data(); } else { *len = item->_data->_indexlen; return item->_data->_index; @@ -221,7 +221,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const case search::ParseItem::ITEM_PURE_WEIGHTED_STRING: { vespalib::stringref term = iterator.getTerm(); - v->VisitKeyword(&item, term.c_str(), term.size(), false, isSpecialToken); + v->VisitKeyword(&item, term.data(), term.size(), false, isSpecialToken); } break; case search::ParseItem::ITEM_NUMTERM: @@ -257,7 +257,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const case search::ParseItem::ITEM_SUBSTRINGTERM: { vespalib::stringref term = iterator.getTerm(); - v->VisitKeyword(&item, term.c_str(), term.size(), true, isSpecialToken); + v->VisitKeyword(&item, term.data(), term.size(), true, isSpecialToken); } break; case search::ParseItem::ITEM_ANY: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h b/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h index 62f2510c17b..15a367761ec 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h @@ -37,7 +37,7 @@ public: _text(textBegin, textEnd - textBegin), _stem(stemBegin, stemEnd - stemBegin), _type(type) {} const vespalib::stringref & getText() const { return _text; } const vespalib::stringref & getStem() const { return _stem; } - bool hasStem() const { return _stem.c_str() != NULL; } + bool hasStem() const { return _stem.data() != NULL; } Type getType() const { return _type; } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index 3a60db52cf3..d75ca47dd33 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -195,7 +195,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const phraseterms_was_added = true; } - keywords.append(term.c_str(), term.size()); + keywords.append(term.data(), term.size()); } } } @@ -218,7 +218,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const vespalib::stringref term = si.getTerm(); if ( !term.empty() && useful(creator)) { // An actual string to add - keywords.append(term.c_str(), term.size()); + keywords.append(term.data(), term.size()); keywords.append("\0", 1); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index 7097012c5ea..9748bdac3b3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -47,7 +47,7 @@ RankFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *s featureDump(json, names[i], values[i]); } json.endObject(); - vespalib::Memory value(json.toString().c_str(), + vespalib::Memory value(json.toString().data(), json.toString().size()); if (type == RES_STRING || type == RES_LONG_STRING) { target.insertString(value); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index c21d6c40066..a5fb9bd8539 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -68,7 +68,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState json.appendDouble(0.0); } json.endObject(); - vespalib::Memory value(json.toString().c_str(), json.toString().size()); + vespalib::Memory value(json.toString().data(), json.toString().size()); if (type == RES_STRING || type == RES_LONG_STRING) { target.insertString(value); } diff --git a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp index f6427f8acd2..7d047e36566 100644 --- a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp +++ b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -40,20 +40,20 @@ MemoryDataStoreTest::testVariableSizeVector() for (size_t i(0); i < 10000; i++) { asciistream os; os << i; - v.push_back(os.str().c_str(), os.str().size()); + v.push_back(os.str().data(), os.str().size()); } for (size_t i(0); i < v.size(); i++) { asciistream os; os << i; EXPECT_EQUAL(os.str().size(), v[i].size()); - EXPECT_EQUAL(0, memcmp(os.str().c_str(), v[i].data(), os.str().size())); + EXPECT_EQUAL(0, memcmp(os.str().data(), v[i].data(), os.str().size())); } size_t i(0); for (auto it(v.begin()), mt(v.end()); it != mt; it++, i++) { asciistream os; os << i; EXPECT_EQUAL(os.str().size(), it->size()); - EXPECT_EQUAL(0, memcmp(os.str().c_str(), (*it).data(), os.str().size())); + EXPECT_EQUAL(0, memcmp(os.str().data(), (*it).data(), os.str().size())); } } diff --git a/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h index cdb9ae95c72..d10687a36ad 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h +++ b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h @@ -9,6 +9,7 @@ class IFieldBase { public: virtual ~IFieldBase() { } + // Overrides must guarantee that returned reference is zero-terminated. virtual stringref getName() const = 0; }; diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h index dfa59e40d7f..4a87dde82e4 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h @@ -171,7 +171,7 @@ public: virtual void loadClass(unsigned classId) = 0; virtual void loadClass(const char * className) = 0; }; - struct RuntimeClass : public IFieldBase { + struct RuntimeClass final : public IFieldBase { public: RuntimeClass(RuntimeInfo * info); ~RuntimeClass(); diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 53875fe7241..22064864b22 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -72,7 +72,7 @@ void GrowableByteBuffer::putString(const vespalib::stringref& v) { putInt(v.size()); - putBytes(v.c_str(), v.size()); + putBytes(v.data(), v.size()); } void diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp index ebeda4f1b8b..0ad52f9aac2 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp @@ -177,7 +177,7 @@ JSONWriter::appendKey(const vespalib::stringref & str) { considerComma(); indent(); - quote(str.c_str(), str.size()); + quote(str.data(), str.size()); (*_os) << ':'; _comma = false; return *this; @@ -246,7 +246,7 @@ JSONWriter & JSONWriter::appendString(const vespalib::stringref & str) { considerComma(); - quote(str.c_str(), str.size()); + quote(str.data(), str.size()); updateCommaState(); return *this; } diff --git a/storage/src/tests/bucketdb/bucketinfotest.cpp b/storage/src/tests/bucketdb/bucketinfotest.cpp index 3eb8d60befd..0298c50866c 100644 --- a/storage/src/tests/bucketdb/bucketinfotest.cpp +++ b/storage/src/tests/bucketdb/bucketinfotest.cpp @@ -51,14 +51,14 @@ getBucketInfo(std::string nodeList, std::string order) { { vespalib::StringTokenizer tokenizer(order, ","); for (uint32_t i = 0; i < tokenizer.size(); i++) { - ordering.push_back(atoi(tokenizer[i].c_str())); + ordering.push_back(atoi(tokenizer[i].data())); } } vespalib::StringTokenizer tokenizer(nodeList, ","); for (uint32_t i = 0; i < tokenizer.size(); i++) { info.addNode(BucketCopy(0, - atoi(tokenizer[i].c_str()), + atoi(tokenizer[i].data()), api::BucketInfo(1,1,1)), ordering); } diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index 559afffc795..56f88b7f98f 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -1800,7 +1800,7 @@ parseInputData(const std::string& data, for (uint32_t i = 0; i < tokenizer.size(); i++) { vespalib::StringTokenizer tok2(tokenizer[i], ":"); - uint16_t node = atoi(tok2[0].c_str()); + uint16_t node = atoi(tok2[0].data()); state.setNodeReplied(node); auto &pendingTransition = state.getPendingBucketSpaceDbTransition(makeBucketSpace()); @@ -1811,19 +1811,19 @@ parseInputData(const std::string& data, vespalib::StringTokenizer tok4(tok3[j], "/"); pendingTransition.addNodeInfo( - document::BucketId(16, atoi(tok4[0].c_str())), + document::BucketId(16, atoi(tok4[0].data())), BucketCopy( timestamp, node, api::BucketInfo( - atoi(tok4[1].c_str()), - atoi(tok4[2].c_str()), - atoi(tok4[3].c_str()), - atoi(tok4[2].c_str()), - atoi(tok4[3].c_str())))); + atoi(tok4[1].data()), + atoi(tok4[2].data()), + atoi(tok4[3].data()), + atoi(tok4[2].data()), + atoi(tok4[3].data())))); } else { pendingTransition.addNodeInfo( - document::BucketId(16, atoi(tok3[j].c_str())), + document::BucketId(16, atoi(tok3[j].data())), BucketCopy(timestamp, node, api::BucketInfo(3, 3, 3, 3, 3))); diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index ce20546dd44..46c756001d9 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -176,11 +176,11 @@ private: trusted = true; } - uint16_t node = atoi(tokenizer2[0].c_str()); + uint16_t node = atoi(tokenizer2[0].data()); if (tokenizer2[1] == "r") { removedNodes.push_back(node); } else { - uint32_t checksum = atoi(tokenizer2[1].c_str()); + uint32_t checksum = atoi(tokenizer2[1].data()); changedNodes.push_back( BucketCopy( i + 1, diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp index e43161946fb..d3496d0c9f6 100644 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -187,16 +187,16 @@ void DistributorTestUtil::addNodesToBucketDB(const document::Bucket& bucket, con vespalib::StringTokenizer tok2(tokenizer[i], "="); vespalib::StringTokenizer tok3(tok2[1], "/"); - api::BucketInfo info(atoi(tok3[0].c_str()), - atoi(tok3.size() > 1 ? tok3[1].c_str() : tok3[0].c_str()), - atoi(tok3.size() > 2 ? tok3[2].c_str() : tok3[0].c_str())); + api::BucketInfo info(atoi(tok3[0].data()), + atoi(tok3.size() > 1 ? tok3[1].data() : tok3[0].data()), + atoi(tok3.size() > 2 ? tok3[2].data() : tok3[0].data())); size_t flagsIdx = 3; // Meta info override? For simplicity, require both meta count and size if (tok3.size() > 4 && (!tok3[3].empty() && isdigit(tok3[3][0]))) { - info.setMetaCount(atoi(tok3[3].c_str())); - info.setUsedFileSize(atoi(tok3[4].c_str())); + info.setMetaCount(atoi(tok3[3].data())); + info.setUsedFileSize(atoi(tok3[4].data())); flagsIdx = 5; } @@ -211,7 +211,7 @@ void DistributorTestUtil::addNodesToBucketDB(const document::Bucket& bucket, con info.setReady(false); } - uint16_t idx = atoi(tok2[0].c_str()); + uint16_t idx = atoi(tok2[0].data()); BucketCopy node( 0, idx, diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index b43d3cf64ad..5551d0a5010 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -572,8 +572,8 @@ parseBucketInfoString(const std::string& nodeList) { BucketInfo entry; for (uint32_t i = 0; i < tokenizer.size(); i++) { vespalib::StringTokenizer tokenizer2(tokenizer[i], "-"); - int node = atoi(tokenizer2[0].c_str()); - int size = atoi(tokenizer2[1].c_str()); + int node = atoi(tokenizer2[0].data()); + int size = atoi(tokenizer2[1].data()); bool trusted = (tokenizer2[2] == "true"); entry.addNode(BucketCopy(0, diff --git a/storage/src/tests/storageserver/fnet_listener_test.cpp b/storage/src/tests/storageserver/fnet_listener_test.cpp index cc9c424ac28..84051041d25 100644 --- a/storage/src/tests/storageserver/fnet_listener_test.cpp +++ b/storage/src/tests/storageserver/fnet_listener_test.cpp @@ -135,7 +135,7 @@ vespalib::string make_compressable_state_string() { ss << " ." << i << ".s:d"; } return vespalib::make_string("version:123 distributor:100%s storage:100%s", - ss.str().c_str(), ss.str().c_str()); + ss.str().data(), ss.str().data()); } } diff --git a/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp b/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp index 0ff178c3f0f..c8d73737d7c 100644 --- a/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp +++ b/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp @@ -65,8 +65,9 @@ uint32_t getTokenCount(const vespalib::string& line) { uint64_t toLong(const vespalib::stringref& s, int base) { char* endptr; - uint64_t result(strtoull(s.c_str(), &endptr, base)); - if ((s.c_str() + s.size()) != endptr) { + // FIXME C++17 range-safe from_chars() instead of strtoull() + uint64_t result(strtoull(s.data(), &endptr, base)); + if ((s.data() + s.size()) != endptr) { throw vespalib::IllegalArgumentException("Parsing '" + s + "' as a long."); } return result; diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index 7c6a708fd8b..e1b7d9c52ea 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -169,7 +169,7 @@ namespace { } else if (state != DeadLockDetector::OK) { vespalib::asciistream ost; ost << "Thread " << id << " has registered tick again.\n"; - LOGBP(info, "%s", ost.str().c_str()); + LOGBP(info, "%s", ost.str().data()); state = DeadLockDetector::OK; } } @@ -200,7 +200,7 @@ DeadLockDetector::handleDeadlock(const framework::MilliSecTime& currentTime, if (warnOnly) { if (_enableWarning) { LOGBT(warning, "deadlockw-" + id, "%s", - error.str().c_str()); + error.str().data()); if (_reportedBucketDBLocksAtState != WARNED) { _reportedBucketDBLocksAtState = WARNED; LOG(info, "Locks in bucket database at deadlock time:" @@ -212,7 +212,7 @@ DeadLockDetector::handleDeadlock(const framework::MilliSecTime& currentTime, } else { if (_enableShutdown || _enableWarning) { LOGBT(error, "deadlock-" + id, "%s", - error.str().c_str()); + error.str().data()); } } if (!_enableShutdown) return; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 9d4d7223411..961af1532a1 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -432,7 +432,7 @@ FileStorManager::onDeleteBucket(const shared_ptr<api::DeleteBucketCommand>& cmd) << ", but storage bucket database contains " << entry->getBucketInfo().toString(); - LOG(debug, "Rejecting bucket delete: %s", ost.str().c_str()); + LOG(debug, "Rejecting bucket delete: %s", ost.str().data()); std::shared_ptr<api::StorageReply> reply = cmd->makeReply(); static_cast<api::DeleteBucketReply&>(*reply).setBucketInfo(entry->getBucketInfo()); reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, ost.str())); diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 370f1c85241..22af1a73633 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -787,7 +787,7 @@ MergeThrottler::validateNewMerge( oss << mergeCmd.toString() << " sent to node " << _component.getIndex() << ", which is not in its forwarding chain"; - LOG(error, "%s", oss.str().c_str()); + LOG(error, "%s", oss.str().data()); } else if (mergeCmd.getChain().size() >= nodeSeq.getSortedNodes().size()) { // Chain is full but we haven't seen the merge! This means // the node has probably gone down with a merge it previously @@ -795,12 +795,12 @@ MergeThrottler::validateNewMerge( oss << mergeCmd.toString() << " is not in node's internal state, but has a " << "full chain, meaning it cannot be forwarded."; - LOG(debug, "%s", oss.str().c_str()); + LOG(debug, "%s", oss.str().data()); } else if (nodeSeq.chainContainsIndex(nodeSeq.getThisNodeIndex())) { oss << mergeCmd.toString() << " is not in node's internal state, but contains " << "this node in its non-full chain. This should not happen!"; - LOG(error, "%s", oss.str().c_str()); + LOG(error, "%s", oss.str().data()); } else { valid = true; } @@ -1117,7 +1117,7 @@ MergeThrottler::makeAbortReply(api::StorageCommand& cmd, vespalib::stringref reason) const { LOG(debug, "Aborting message %s with reason '%s'", - cmd.toString().c_str(), reason.c_str()); + cmd.toString().c_str(), reason.data()); std::unique_ptr<api::StorageReply> reply(cmd.makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, reason)); return std::shared_ptr<api::StorageMessage>(reply.release()); diff --git a/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp b/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp index 41177fe46b8..e26549a3b37 100644 --- a/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp +++ b/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp @@ -15,21 +15,21 @@ void ServiceLayerErrorListener::on_fatal_error(vespalib::stringref message) { LOG(info, "Received FATAL_ERROR from persistence provider, " "shutting down node: %s", - message.c_str()); + vespalib::string(message).c_str()); _component.requestShutdown(message); // Thread safe } else { LOG(debug, "Received FATAL_ERROR from persistence provider: %s. " "Node has already been instructed to shut down so " "not doing anything now.", - message.c_str()); + vespalib::string(message).c_str()); } } void ServiceLayerErrorListener::on_resource_exhaustion_error(vespalib::stringref message) { LOG(debug, "SPI reports resource exhaustion ('%s'). " "Applying back-pressure to merge throttler", - message.c_str()); + vespalib::string(message).c_str()); _merge_throttler.apply_timed_backpressure(); // Thread safe } diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index d6ca8c266d3..77d9299169f 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -375,7 +375,7 @@ considerInsertDerivedTransition(const lib::State ¤tBaseline, ((currentDerived != currentBaseline) || (newDerived != newBaseline))); if (considerDerivedTransition && (transitions.find(bucketSpace) == transitions.end())) { transitions[bucketSpace] = vespalib::make_string("%s space: '%s' to '%s'", - document::FixedBucketSpaces::to_string(bucketSpace).c_str(), + document::FixedBucketSpaces::to_string(bucketSpace).data(), currentDerived.getName().c_str(), newDerived.getName().c_str()); } diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index b12a1eb6e7f..6bf28b08540 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -253,7 +253,7 @@ VisitorThread::run(framework::ThreadHandle& thread) } catch (std::exception& e) { vespalib::asciistream ost; ost << "Failed to handle visitor message:" << e.what(); - LOG(warning, "Failed handling visitor message: %s", ost.str().c_str()); + LOG(warning, "Failed handling visitor message: %s", ost.str().data()); result = ReturnCode(ReturnCode::INTERNAL_FAILURE, ost.str()); if (entry._message.get() && entry._message->getType() == api::MessageType::VISITOR_CREATE) { _messageSender.closed(entry._visitorId); @@ -466,7 +466,7 @@ VisitorThread::onCreateVisitor( if (visitor.get() == 0) { result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, errors.str()); LOG(warning, "CreateVisitor(%s): Failed to create visitor: %s", - cmd->getInstanceId().c_str(), errors.str().c_str()); + cmd->getInstanceId().c_str(), errors.str().data()); break; } // Set visitor parameters @@ -510,7 +510,7 @@ VisitorThread::onCreateVisitor( << cmd->getDocumentSelection() << "': " << e.getMessage(); result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, ost.str()); LOG(warning, "CreateVisitor(%s): %s", - cmd->getInstanceId().c_str(), ost.str().c_str()); + cmd->getInstanceId().c_str(), ost.str().data()); break; } catch (document::select::ParsingFailedException& e) { vespalib::asciistream ost; @@ -518,7 +518,7 @@ VisitorThread::onCreateVisitor( << cmd->getDocumentSelection() << "': " << e.getMessage(); result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, ost.str()); LOG(warning, "CreateVisitor(%s): %s", - cmd->getInstanceId().c_str(), ost.str().c_str()); + cmd->getInstanceId().c_str(), ost.str().data()); break; } LOG(debug, "CreateVisitor(%s): Successfully created visitor", diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index a5611451ee8..ae448664f14 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -132,7 +132,7 @@ namespace { MetricHookWrapper(vespalib::stringref name, MetricUpdateHook& hook) - : metrics::UpdateHook(name.c_str()), + : metrics::UpdateHook(name.data()), // Expected to point to static name _hook(hook) { } diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index 3090910d2bb..7af48c1904b 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -191,7 +191,7 @@ namespace { try{ std::vector<uint16_t> nvect; distribution.getIdealNodes(nodeType, state, results[i].bucket, - nvect, upStates.c_str(), redundancy); + nvect, upStates.data(), redundancy); IdealNodeList nodes; for (uint32_t j=0, m=nvect.size(); j<m; ++j) { nodes.push_back(Node(nodeType, nvect[j])); diff --git a/vdslib/src/tests/distribution/grouptest.cpp b/vdslib/src/tests/distribution/grouptest.cpp index c108a1e8a38..71d8b8bac0c 100644 --- a/vdslib/src/tests/distribution/grouptest.cpp +++ b/vdslib/src/tests/distribution/grouptest.cpp @@ -30,7 +30,7 @@ namespace { vespalib::StringTokenizer st(nodelist, ","); std::vector<uint16_t> nodes(st.size()); for (uint32_t i=0; i<st.size(); ++i) { - nodes[i] = atoi(st[i].c_str()); + nodes[i] = atoi(st[i].data()); } group->setNodes(nodes); return group; diff --git a/vdslib/src/vespa/vdslib/container/parameters.h b/vdslib/src/vespa/vdslib/container/parameters.h index 4572b2f6f1c..ab65932496a 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.h +++ b/vdslib/src/vespa/vdslib/container/parameters.h @@ -68,7 +68,7 @@ public: ParametersMap::const_iterator begin() const { return _parameters.begin(); } ParametersMap::const_iterator end() const { return _parameters.end(); } /// Convenience from earlier use. - void set(const KeyT & id, const vespalib::stringref & value) { _parameters[id] = Value(value.c_str(), value.size()); } + void set(const KeyT & id, const vespalib::stringref & value) { _parameters[id] = Value(value.data(), value.size()); } vespalib::stringref get(const KeyT & id, const vespalib::stringref & def = "") const; /** * Set the value identified by the id given. This requires the type to be diff --git a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp index f620dc15928..99974225e9e 100644 --- a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp @@ -30,7 +30,7 @@ namespace { firstAsterisk = i; continue; } - uint32_t number = atoi(st[i].c_str()); + uint32_t number = atoi(vespalib::string(st[i]).c_str()); if (number <= 0 || number >= 256) { throw vespalib::IllegalArgumentException( "Illegal distribution spec \"" + serialized + "\". " @@ -48,7 +48,7 @@ namespace { } } - std::vector<uint16_t> parse(vespalib::stringref& serialized) { + std::vector<uint16_t> parse(vespalib::stringref serialized) { std::vector<uint16_t> result; if (serialized == "") return result; vespalib::StringTokenizer st(serialized, "|"); diff --git a/vdslib/src/vespa/vdslib/state/clusterstate.cpp b/vdslib/src/vespa/vdslib/state/clusterstate.cpp index fbc9943e22d..e50b18a5a82 100644 --- a/vdslib/src/vespa/vdslib/state/clusterstate.cpp +++ b/vdslib/src/vespa/vdslib/state/clusterstate.cpp @@ -88,7 +88,7 @@ ClusterState::ClusterState(const vespalib::string& serialized) if (key.empty() || ! parse(key, value, nodeData) ) { LOG(debug, "Unknown key %s in systemstate. Ignoring it, assuming it's " "a new feature from a newer version than ourself: %s", - key.c_str(), serialized.c_str()); + vespalib::string(key).c_str(), serialized.c_str()); } } nodeData.addTo(_nodeStates, _nodeCount); @@ -106,13 +106,13 @@ ClusterState::parse(vespalib::stringref key, vespalib::stringref value, NodeData break; case 'b': if (key == "bits") { - _distributionBits = atoi(value.c_str()); + _distributionBits = atoi(value.data()); return true; } break; case 'v': if (key == "version") { - _version = atoi(value.c_str()); + _version = atoi(value.data()); return true; } break; @@ -145,7 +145,7 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node if (nodeType == 0) return false; if (dot == vespalib::string::npos) { // Entry that set node counts uint16_t nodeCount = 0; - nodeCount = atoi(value.c_str()); + nodeCount = atoi(value.data()); if (nodeCount > _nodeCount[*nodeType] ) { _nodeCount[*nodeType] = nodeCount; @@ -155,9 +155,9 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node vespalib::string::size_type dot2 = key.find('.', dot + 1); Node node; if (dot2 == vespalib::string::npos) { - node = Node(*nodeType, atoi(key.substr(dot + 1).c_str())); + node = Node(*nodeType, atoi(key.substr(dot + 1).data())); } else { - node = Node(*nodeType, atoi(key.substr(dot + 1, dot2 - dot - 1).c_str())); + node = Node(*nodeType, atoi(key.substr(dot + 1, dot2 - dot - 1).data())); } if (node.getIndex() >= _nodeCount[*nodeType]) { diff --git a/vdslib/src/vespa/vdslib/state/clusterstate.h b/vdslib/src/vespa/vdslib/state/clusterstate.h index 9e8dd0f292a..26c6f1b95ef 100644 --- a/vdslib/src/vespa/vdslib/state/clusterstate.h +++ b/vdslib/src/vespa/vdslib/state/clusterstate.h @@ -72,7 +72,9 @@ public: const std::string& indent = "") const; private: + // Preconditions: `key` and `value` MUST point into null-terminated strings. bool parse(vespalib::stringref key, vespalib::stringref value, NodeData & nodeData); + // Preconditions: `key` and `value` MUST point into null-terminated strings. bool parseSorD(vespalib::stringref key, vespalib::stringref value, NodeData & nodeData); void removeExtraElements(); void printStateGroupwise(std::ostream& out, const Group&, bool verbose, diff --git a/vdslib/src/vespa/vdslib/state/diskstate.cpp b/vdslib/src/vespa/vdslib/state/diskstate.cpp index bcc380e0d75..c963dacff82 100644 --- a/vdslib/src/vespa/vdslib/state/diskstate.cpp +++ b/vdslib/src/vespa/vdslib/state/diskstate.cpp @@ -73,7 +73,7 @@ DiskState::DiskState(const vespalib::stringref & serialized) } LOG(debug, "Unknown key %s in diskstate. Ignoring it, assuming it's a " "new feature from a newer version than ourself: %s", - key.c_str(), serialized.c_str()); + key.c_str(), vespalib::string(serialized).c_str()); } } diff --git a/vdslib/src/vespa/vdslib/state/nodestate.cpp b/vdslib/src/vespa/vdslib/state/nodestate.cpp index ff6c8f31c89..d59686dcb1c 100644 --- a/vdslib/src/vespa/vdslib/state/nodestate.cpp +++ b/vdslib/src/vespa/vdslib/state/nodestate.cpp @@ -229,7 +229,7 @@ NodeState::NodeState(const vespalib::stringref & serialized, const NodeType* typ } LOG(debug, "Unknown key %s in nodestate. Ignoring it, assuming it's a " "new feature from a newer version than ourself: %s", - key.c_str(), serialized.c_str()); + key.c_str(), vespalib::string(serialized).c_str()); } diskData.addTo(_diskStates); updateAnyDiskDownFlag(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPublicKey.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPublicKey.java deleted file mode 100644 index 1c810e3e9c9..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPublicKey.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.api; - -import java.security.PublicKey; -import java.util.Objects; - -/** - * @author bjorncs - */ -public class AthenzPublicKey { - - private final PublicKey publicKey; - private final String keyId; - - public AthenzPublicKey(PublicKey publicKey, String keyId) { - this.publicKey = publicKey; - this.keyId = keyId; - } - - public PublicKey getPublicKey() { - return publicKey; - } - - public String getKeyId() { - return keyId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AthenzPublicKey that = (AthenzPublicKey) o; - return Objects.equals(publicKey, that.publicKey) && - Objects.equals(keyId, that.keyId); - } - - @Override - public int hashCode() { - return Objects.hash(publicKey, keyId); - } - - @Override - public String toString() { - return "AthenzPublicKey{" + - "publicKey=" + publicKey + - ", keyId='" + keyId + '\'' + - '}'; - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java index 563cae80da2..c2be1a40893 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.athenz.tls; import com.yahoo.athenz.auth.util.Crypto; +import org.bouncycastle.asn1.ASN1Encodable; +import org.bouncycastle.asn1.ASN1Primitive; import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.openssl.PEMKeyPair; import org.bouncycastle.openssl.PEMParser; @@ -70,11 +72,21 @@ public class KeyUtils { public static String toPem(PrivateKey privateKey) { try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { - pemWriter.writeObject(new PemObject("PRIVATE KEY", privateKey.getEncoded())); + // Note: Encoding using PKCS#1 as this is to be read by tools only supporting PKCS#1 + pemWriter.writeObject(new PemObject("RSA PRIVATE KEY", getPkcs1Bytes(privateKey))); pemWriter.flush(); return stringWriter.toString(); } catch (IOException e) { throw new UncheckedIOException(e); } } + + private static byte[] getPkcs1Bytes(PrivateKey privateKey) throws IOException{ + + byte[] privBytes = privateKey.getEncoded(); + PrivateKeyInfo pkInfo = PrivateKeyInfo.getInstance(privBytes); + ASN1Encodable encodable = pkInfo.parsePrivateKey(); + ASN1Primitive primitive = encodable.toASN1Primitive(); + return primitive.getEncoded(); + } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzConfTruststore.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzConfTruststore.java new file mode 100644 index 00000000000..4cb3470635e --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzConfTruststore.java @@ -0,0 +1,57 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.utils.ntoken; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.yahoo.athenz.auth.util.Crypto; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.security.PublicKey; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * A {@link AthenzTruststore} that is backed by athenz.conf + * + * @author bjorncs + */ +public class AthenzConfTruststore implements AthenzTruststore { + + private final Map<String, PublicKey> zmsPublicKeys; + private final Map<String, PublicKey> ztsPublicKeys; + + public AthenzConfTruststore(Path athenzConfFile) { + try { + JsonNode root = new ObjectMapper().readTree(athenzConfFile.toFile()); + this.zmsPublicKeys = loadPublicKeys((ArrayNode) root.get("zmsPublicKeys")); + this.ztsPublicKeys = loadPublicKeys((ArrayNode) root.get("ztsPublicKeys")); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static Map<String, PublicKey> loadPublicKeys(ArrayNode keysArray) { + Map<String, PublicKey> publicKeys = new HashMap<>(); + for (JsonNode keyEntry : keysArray) { + String keyId = keyEntry.get("id").textValue(); + String encodedPublicKey = keyEntry.get("key").textValue(); + PublicKey publicKey = Crypto.loadPublicKey(Crypto.ybase64DecodeString(encodedPublicKey)); + publicKeys.put(keyId, publicKey); + } + return publicKeys; + } + + @Override + public Optional<PublicKey> getZmsPublicKey(String keyId) { + return Optional.ofNullable(zmsPublicKeys.get(keyId)); + } + + @Override + public Optional<PublicKey> getZtsPublicKey(String keyId) { + return Optional.ofNullable(ztsPublicKeys.get(keyId)); + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzTruststore.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzTruststore.java new file mode 100644 index 00000000000..83afa288cf0 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzTruststore.java @@ -0,0 +1,15 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.utils.ntoken; + +import java.security.PublicKey; +import java.util.Optional; + +/** + * A truststore that contains all ZMS and ZTS public keys + * + * @author bjorncs + */ +public interface AthenzTruststore { + Optional<PublicKey> getZmsPublicKey(String keyId); + Optional<PublicKey> getZtsPublicKey(String keyId); +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidator.java new file mode 100644 index 00000000000..f4ec0b168d7 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidator.java @@ -0,0 +1,72 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.utils.ntoken; + +import com.yahoo.athenz.auth.token.PrincipalToken; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzPrincipal; +import com.yahoo.vespa.athenz.api.NToken; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; + +import java.nio.file.Path; +import java.security.PublicKey; +import java.time.Duration; +import java.util.logging.Logger; + +/** + * Validates the content of an NToken: + * 1) Verifies that the token is signed by Athenz + * 2) Verifies that the token is not expired + * + * @author bjorncs + */ +public class NTokenValidator { + // Max allowed skew in token timestamp (only for creation, not expiry timestamp) + private static final long ALLOWED_TIMESTAMP_OFFSET = Duration.ofMinutes(5).getSeconds(); + + private static final Logger log = Logger.getLogger(NTokenValidator.class.getName()); + private final AthenzTruststore truststore; + + + public NTokenValidator(AthenzTruststore truststore) { + this.truststore = truststore; + } + + public NTokenValidator(Path athenzConfFile) { + this(new AthenzConfTruststore(athenzConfFile)); + } + + public AthenzPrincipal validate(NToken token) throws InvalidTokenException { + PrincipalToken principalToken = new PrincipalToken(token.getRawToken()); + String keyId = principalToken.getKeyId(); + String keyService = principalToken.getKeyService(); + PublicKey zmsPublicKey = (keyService == null || keyService.equals("zms") ? truststore.getZmsPublicKey(keyId) : truststore.getZtsPublicKey(keyId)) + .orElseThrow(() -> { + String message = "NToken has an unknown keyId: " + keyId; + log.log(LogLevel.WARNING, message); + return new InvalidTokenException(message); + }); + validateSignatureAndExpiration(principalToken, zmsPublicKey); + return new AthenzPrincipal( + AthenzIdentities.from( + new AthenzDomain(principalToken.getDomain()), + principalToken.getName()), + token); + } + + private static void validateSignatureAndExpiration(PrincipalToken token, PublicKey zmsPublicKey) throws InvalidTokenException { + StringBuilder errorMessageBuilder = new StringBuilder(); + if (!token.validate(zmsPublicKey, (int) ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { + String message = "NToken is expired or has invalid signature: " + errorMessageBuilder.toString(); + log.log(LogLevel.WARNING, message); + throw new InvalidTokenException(message); + } + } + + public static class InvalidTokenException extends RuntimeException { + public InvalidTokenException(String message) { + super(message); + } + } + +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/package-info.java new file mode 100644 index 00000000000..8760c02d27d --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author bjorncs + */ +@ExportPackage +package com.yahoo.vespa.athenz.utils.ntoken; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java index fca4353d400..fbdc6f1e3bd 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java @@ -27,8 +27,8 @@ public class KeyUtilsTest { public void can_serialize_deserialize_pem() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); String pem = KeyUtils.toPem(keyPair.getPrivate()); - assertThat(pem, containsString("BEGIN PRIVATE KEY")); - assertThat(pem, containsString("END PRIVATE KEY")); + assertThat(pem, containsString("BEGIN RSA PRIVATE KEY")); + assertThat(pem, containsString("END RSA PRIVATE KEY")); PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); assertEquals(keyPair.getPrivate(), deserializedKey); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidatorTest.java index 510c806383c..22f97ca8b60 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidatorTest.java @@ -1,23 +1,24 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz.filter; +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.utils.ntoken; import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.NToken; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; +import com.yahoo.vespa.athenz.tls.KeyAlgorithm; +import com.yahoo.vespa.athenz.tls.KeyUtils; +import com.yahoo.vespa.athenz.utils.ntoken.NTokenValidator.InvalidTokenException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.security.KeyPair; import java.security.PrivateKey; +import java.security.PublicKey; import java.time.Instant; import java.util.Optional; -import static com.yahoo.vespa.athenz.utils.AthenzIdentities.ZMS_ATHENZ_SERVICE; import static org.junit.Assert.assertEquals; /** @@ -25,8 +26,8 @@ import static org.junit.Assert.assertEquals; */ public class NTokenValidatorTest { - private static final KeyPair TRUSTED_KEY = AthenzTestUtils.generateRsaKeypair(); - private static final KeyPair UNKNOWN_KEY = AthenzTestUtils.generateRsaKeypair(); + private static final KeyPair TRUSTED_KEY = KeyUtils.generateKeypair(KeyAlgorithm.RSA); + private static final KeyPair UNKNOWN_KEY = KeyUtils.generateKeypair(KeyAlgorithm.RSA); private static final AthenzIdentity IDENTITY = AthenzUser.fromUserId("myuser"); @Rule @@ -34,7 +35,7 @@ public class NTokenValidatorTest { @Test public void valid_token_is_accepted() throws InvalidTokenException { - NTokenValidator validator = new NTokenValidator(createKeystore()); + NTokenValidator validator = new NTokenValidator(createTruststore()); NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0"); AthenzPrincipal principal = validator.validate(token); assertEquals("user.myuser", principal.getIdentity().getFullName()); @@ -42,7 +43,7 @@ public class NTokenValidatorTest { @Test public void invalid_signature_is_not_accepted() throws InvalidTokenException { - NTokenValidator validator = new NTokenValidator(createKeystore()); + NTokenValidator validator = new NTokenValidator(createTruststore()); NToken token = createNToken(IDENTITY, Instant.now(), UNKNOWN_KEY.getPrivate(), "0"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("NToken is expired or has invalid signature"); @@ -51,7 +52,7 @@ public class NTokenValidatorTest { @Test public void expired_token_is_not_accepted() throws InvalidTokenException { - NTokenValidator validator = new NTokenValidator(createKeystore()); + NTokenValidator validator = new NTokenValidator(createTruststore()); NToken token = createNToken(IDENTITY, Instant.ofEpochMilli(1234) /*long time ago*/, TRUSTED_KEY.getPrivate(), "0"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("NToken is expired or has invalid signature"); @@ -60,28 +61,25 @@ public class NTokenValidatorTest { @Test public void unknown_keyId_is_not_accepted() throws InvalidTokenException { - NTokenValidator validator = new NTokenValidator(createKeystore()); + NTokenValidator validator = new NTokenValidator(createTruststore()); NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "unknown-key-id"); exceptionRule.expect(InvalidTokenException.class); exceptionRule.expectMessage("NToken has an unknown keyId"); validator.validate(token); } - @Test - public void failing_to_find_key_should_throw_exception() throws InvalidTokenException { - ZmsKeystore keystore = (athensService, keyId) -> { throw new RuntimeException(); }; - NTokenValidator validator = new NTokenValidator(keystore); - NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0"); - exceptionRule.expect(InvalidTokenException.class); - exceptionRule.expectMessage("Failed to retrieve public key"); - validator.validate(token); - } + private static AthenzTruststore createTruststore() { + return new AthenzTruststore() { + @Override + public Optional<PublicKey> getZmsPublicKey(String keyId) { + return keyId.equals("0") ? Optional.of(TRUSTED_KEY.getPublic()) : Optional.empty(); + } - private static ZmsKeystore createKeystore() { - return (athensService, keyId) -> - athensService.equals(ZMS_ATHENZ_SERVICE) && keyId.equals("0") - ? Optional.of(TRUSTED_KEY.getPublic()) - : Optional.empty(); + @Override + public Optional<PublicKey> getZtsPublicKey(String keyId) { + return Optional.empty(); + } + }; } private static NToken createNToken(AthenzIdentity identity, Instant issueTime, PrivateKey privateKey, String keyId) { @@ -90,6 +88,7 @@ public class NTokenValidatorTest { .salt("1234") .host("host") .ip("1.2.3.4") + .keyService("zms") .issueTime(issueTime.getEpochSecond()) .expirationWindow(1000) .build(); diff --git a/vespalib/src/tests/stllike/string_test.cpp b/vespalib/src/tests/stllike/string_test.cpp index 2973ffd1ef1..96c095667ee 100644 --- a/vespalib/src/tests/stllike/string_test.cpp +++ b/vespalib/src/tests/stllike/string_test.cpp @@ -265,10 +265,10 @@ TEST("testString") { // Test std::string conversion of empty string stringref sref; std::string stdString(sref); - EXPECT_TRUE(strcmp("", sref.c_str()) == 0); + EXPECT_TRUE(strcmp("", sref.data()) == 0); stdString = "abc"; stringref sref2(stdString); - EXPECT_TRUE(stdString.c_str() == sref2.c_str()); + EXPECT_TRUE(stdString.c_str() == sref2.data()); EXPECT_TRUE(stdString == sref2); EXPECT_TRUE(sref2 == stdString); { diff --git a/vespalib/src/vespa/vespalib/component/version.cpp b/vespalib/src/vespa/vespalib/component/version.cpp index af38a675de8..3aa8e134e36 100644 --- a/vespalib/src/vespa/vespalib/component/version.cpp +++ b/vespalib/src/vespa/vespalib/component/version.cpp @@ -62,10 +62,11 @@ Version::verifySanity() } } +// Precondition: input.empty() == false static int parseInteger(const stringref & input) __attribute__((noinline)); static int parseInteger(const stringref & input) { - const char *s = input.c_str(); + const char *s = input.data(); unsigned char firstDigit = s[0]; if (!isdigit(firstDigit)) throw IllegalArgumentException("integer must start with a digit"); diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index 389218cea35..e360c84f569 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -19,7 +19,7 @@ namespace vespalib { namespace { FileInfo::UP - processStat(struct stat& filestats, bool result, const stringref & path) { + processStat(struct stat& filestats, bool result, stringref path) { FileInfo::UP resval; if (result) { resval.reset(new FileInfo); @@ -36,7 +36,7 @@ namespace { } LOG(debug, "stat(%s): Existed? %s, Plain file? %s, Directory? %s, " "Size: %" PRIu64, - path.c_str(), + string(path).c_str(), resval.get() ? "true" : "false", resval.get() && resval->_plainfile ? "true" : "false", resval.get() && resval->_directory ? "true" : "false", @@ -69,7 +69,7 @@ operator<<(std::ostream& out, const FileInfo& info) return out; } -File::File(const stringref & filename) +File::File(stringref filename) : _fd(-1), _flags(0), _filename(filename), @@ -79,7 +79,7 @@ File::File(const stringref & filename) { } -File::File(int fileDescriptor, const stringref & filename) +File::File(int fileDescriptor, stringref filename) : _fd(fileDescriptor), _flags(0), _filename(filename), @@ -128,7 +128,7 @@ File::operator=(File& f) } void -File::setFilename(const stringref & filename) +File::setFilename(stringref filename) { if (_filename == filename) return; if (_close && _fd != -1) close(); @@ -139,14 +139,14 @@ File::setFilename(const stringref & filename) } namespace { - int openAndCreateDirsIfMissing(const stringref & filename, int flags, + int openAndCreateDirsIfMissing(const string & filename, int flags, bool createDirsIfMissing) { int fd = ::open(filename.c_str(), flags, 0644); if (fd < 0 && errno == ENOENT && ((flags & O_CREAT) != 0) && createDirsIfMissing) { - string::size_type pos = filename.rfind('/'); + auto pos = filename.rfind('/'); if (pos != string::npos) { string path(filename.substr(0, pos)); mkdir(path); @@ -381,7 +381,7 @@ File::readAll() const } vespalib::string -File::readAll(const vespalib::stringref & path) +File::readAll(vespalib::stringref path) { File file(path); file.open(File::READONLY); @@ -452,14 +452,14 @@ getCurrentDirectory() } bool -mkdir(const stringref & directory, bool recursive) +mkdir(const string & directory, bool recursive) { if (::mkdir(directory.c_str(), 0777) == 0) { LOG(debug, "mkdir(%s): Created directory", directory.c_str()); return true; } if (recursive && errno == ENOENT) { - string::size_type slashpos = directory.rfind('/'); + auto slashpos = directory.rfind('/'); if (slashpos != string::npos) { /* Recursively make superdirs.*/ string superdir = directory.substr(0, slashpos); @@ -499,8 +499,8 @@ mkdir(const stringref & directory, bool recursive) } void -symlink(const stringref & oldPath, - const stringref & newPath) +symlink(const string & oldPath, + const string & newPath) { if (::symlink(oldPath.c_str(), newPath.c_str())) { asciistream ss; @@ -513,7 +513,7 @@ symlink(const stringref & oldPath, } string -readLink(const stringref & path) +readLink(const string & path) { char buf[256]; ssize_t bytes(::readlink(path.c_str(), buf, sizeof(buf))); @@ -528,7 +528,7 @@ readLink(const stringref & path) } void -chdir(const stringref & directory) +chdir(const string & directory) { if (::chdir(directory.c_str()) != 0) { asciistream ost; @@ -537,12 +537,11 @@ chdir(const stringref & directory) throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } - LOG(debug, "chdir(%s): Working directory changed.", - directory.c_str()); + LOG(debug, "chdir(%s): Working directory changed.", directory.c_str()); } bool -rmdir(const stringref & directory, bool recursive) +rmdir(const string & directory, bool recursive) { string dirname(directory); if (!dirname.empty() && *dirname.rbegin() == '/') { @@ -595,26 +594,26 @@ rmdir(const stringref & directory, bool recursive) } FileInfo::UP -stat(const stringref & path) +stat(const string & path) { struct ::stat filestats; return processStat(filestats, ::stat(path.c_str(), &filestats) == 0, path); } FileInfo::UP -lstat(const stringref & path) +lstat(const string & path) { struct ::stat filestats; return processStat(filestats, ::lstat(path.c_str(), &filestats) == 0, path); } bool -fileExists(const vespalib::stringref & path) { +fileExists(const string & path) { return (stat(path).get() != 0); } bool -unlink(const stringref & filename) +unlink(const string & filename) { if (::unlink(filename.c_str()) != 0) { if (errno == ENOENT) { @@ -631,7 +630,7 @@ unlink(const stringref & filename) } bool -rename(const stringref & frompath, const stringref & topath, +rename(const string & frompath, const string & topath, bool copyDeleteBetweenFilesystems, bool createTargetDirectoryIfMissing) { LOG(spam, "rename(%s, %s): Renaming file%s.", @@ -696,7 +695,7 @@ namespace { } void -copy(const stringref & frompath, const stringref & topath, +copy(const string & frompath, const string & topath, bool createTargetDirectoryIfMissing, bool useDirectIO) { // Get aligned buffer, so it works with direct IO @@ -730,7 +729,7 @@ copy(const stringref & frompath, const stringref & topath, } DirectoryList -listDirectory(const stringref & path) +listDirectory(const string & path) { DIR* dir = ::opendir(path.c_str()); struct dirent* entry; @@ -762,7 +761,7 @@ getAlignedBuffer(size_t size) return MallocAutoPtr(ptr); } -string dirname(const stringref name) +string dirname(stringref name) { size_t found = name.rfind('/'); if (found == string::npos) { @@ -776,7 +775,7 @@ string dirname(const stringref name) namespace { -void addStat(asciistream &os, const stringref name) +void addStat(asciistream &os, const string & name) { struct ::stat filestat; memset(&filestat, '\0', sizeof(filestat)); @@ -801,7 +800,7 @@ void addStat(asciistream &os, const stringref name) } string -getOpenErrorString(const int osError, const stringref filename) +getOpenErrorString(const int osError, stringref filename) { asciistream os; string dirName(dirname(filename)); @@ -809,12 +808,12 @@ getOpenErrorString(const int osError, const stringref filename) getErrorString(osError) << "\") fileStat"; addStat(os, filename); os << " dirStat"; - addStat(os, dirName.c_str()); + addStat(os, dirName); return os.str(); } bool -isDirectory(const vespalib::stringref & path) { +isDirectory(const string & path) { FileInfo::UP info(stat(path)); return (info.get() && info->_directory); } diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h index ece37ab0108..187939ed412 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.h +++ b/vespalib/src/vespa/vespalib/io/fileutil.h @@ -83,10 +83,10 @@ public: enum Flag { READONLY = 1, CREATE = 2, DIRECTIO = 4, TRUNC = 8 }; /** Create a file instance, without opening the file. */ - File(const vespalib::stringref & filename); + File(vespalib::stringref filename); /** Create a file instance of an already open file. */ - File(int fileDescriptor, const vespalib::stringref & filename); + File(int fileDescriptor, vespalib::stringref filename); /** Copying a file instance, moves any open file descriptor. */ File(File& f); @@ -99,7 +99,7 @@ public: * Make this instance point at another file. * Closes the old file it it was open. */ - void setFilename(const vespalib::stringref & filename); + void setFilename(vespalib::stringref filename); const vespalib::string& getFilename() const { return _filename; } @@ -188,7 +188,7 @@ public: * @throw IoException If we failed to read from file. * @return The content of the file. */ - static vespalib::string readAll(const vespalib::stringref & path); + static vespalib::string readAll(vespalib::stringref path); virtual void sync(); virtual bool close(); @@ -273,7 +273,7 @@ extern vespalib::string getCurrentDirectory(); * * @return True if it did not exist, false if it did. */ -extern bool mkdir(const vespalib::stringref & directory, bool recursive = true); +extern bool mkdir(const vespalib::string & directory, bool recursive = true); /** * Change working directory. @@ -281,7 +281,7 @@ extern bool mkdir(const vespalib::stringref & directory, bool recursive = true); * @param directory The directory to change to. * @throw IoException If we failed to change to the new working directory. */ -extern void chdir(const vespalib::stringref & directory); +extern void chdir(const vespalib::string & directory); /** * Remove a directory. @@ -293,7 +293,7 @@ extern void chdir(const vespalib::stringref & directory); * * @return True if directory existed, false if not. */ -extern bool rmdir(const vespalib::stringref & directory, bool recursive = false); +extern bool rmdir(const vespalib::string & directory, bool recursive = false); /** * Stat a file. @@ -302,7 +302,7 @@ extern bool rmdir(const vespalib::stringref & directory, bool recursive = false) * @return A file info object if everything went well, a null pointer if the * file was not found. */ -extern FileInfo::UP stat(const vespalib::stringref & path); +extern FileInfo::UP stat(const vespalib::string & path); /** * Stat a file. Give info on symlink rather than on file pointed to. @@ -311,14 +311,14 @@ extern FileInfo::UP stat(const vespalib::stringref & path); * @return A file info object if everything went well, a null pointer if the * file was not found. */ -extern FileInfo::UP lstat(const vespalib::stringref & path); +extern FileInfo::UP lstat(const vespalib::string & path); /** * Check if a file exists or not. See also pathExists. * * @throw IoException If we failed to stat the file. */ -extern bool fileExists(const vespalib::stringref & path); +extern bool fileExists(const vespalib::string & path); /** * Check if a path exists, i.e. whether it's a symbolic link, regular file, @@ -328,7 +328,7 @@ extern bool fileExists(const vespalib::stringref & path); * This function returns true, while fileExists returns true only if the path * the symbolic link points to exists. */ -extern inline bool pathExists(const vespalib::stringref & path) { +extern inline bool pathExists(const vespalib::string & path) { return (lstat(path).get() != 0); } @@ -336,7 +336,7 @@ extern inline bool pathExists(const vespalib::stringref & path) { * Get the filesize of the given file. Ignoring if it exists or not. * (None-existing files will be reported to have size zero) */ -extern inline off_t getFileSize(const vespalib::stringref & path) { +extern inline off_t getFileSize(const vespalib::string & path) { FileInfo::UP info(stat(path)); return (info.get() == 0 ? 0 : info->_size); } @@ -347,7 +347,7 @@ extern inline off_t getFileSize(const vespalib::stringref & path) { * @return True if it is a plain file, false if it don't exist or isn't. * @throw IoException If we failed to stat the file. */ -extern inline bool isPlainFile(const vespalib::stringref & path) { +extern inline bool isPlainFile(const vespalib::string & path) { FileInfo::UP info(stat(path)); return (info.get() && info->_plainfile); } @@ -358,7 +358,7 @@ extern inline bool isPlainFile(const vespalib::stringref & path) { * @return True if it is a directory, false if it don't exist or isn't. * @throw IoException If we failed to stat the file. */ -extern bool isDirectory(const vespalib::stringref & path); +extern bool isDirectory(const vespalib::string & path); /** * Check whether a path is a symlink. @@ -366,7 +366,7 @@ extern bool isDirectory(const vespalib::stringref & path); * @return True if path exists and is a symbolic link. * @throw IoException If there's an unexpected stat failure. */ -extern inline bool isSymLink(const vespalib::stringref & path) { +extern inline bool isSymLink(const vespalib::string & path) { FileInfo::UP info(lstat(path)); return (info.get() && info->_symlink); } @@ -384,8 +384,8 @@ extern inline bool isSymLink(const vespalib::stringref & path) { * @param newPath Relative link to be created. See above note for semantics. * @throw IoException if we fail to create the symlink. */ -extern void symlink(const vespalib::stringref & oldPath, - const vespalib::stringref & newPath); +extern void symlink(const vespalib::string & oldPath, + const vespalib::string & newPath); /** * Read and return the contents of symbolic link at the given path. @@ -394,7 +394,7 @@ extern void symlink(const vespalib::stringref & oldPath, * @return Contents of symbolic link. * @throw IoException if we cannot read the link. */ -extern vespalib::string readLink(const vespalib::stringref & path); +extern vespalib::string readLink(const vespalib::string & path); /** * Remove the given file. @@ -403,7 +403,7 @@ extern vespalib::string readLink(const vespalib::stringref & path); * @return True if file was removed, false if it did not exist. * @throw IoException If we failed to unlink the file. */ -extern bool unlink(const vespalib::stringref & filename); +extern bool unlink(const vespalib::string & filename); /** * Rename the file at frompath to topath. @@ -421,16 +421,16 @@ extern bool unlink(const vespalib::stringref & filename); * @throw IoException If we failed to rename the file. * @return True if file was renamed, false if frompath did not exist. */ -extern bool rename(const vespalib::stringref & frompath, - const vespalib::stringref & topath, +extern bool rename(const vespalib::string & frompath, + const vespalib::string & topath, bool copyDeleteBetweenFilesystems = true, bool createTargetDirectoryIfMissing = false); /** * Copies a file to a destination using Direct IO. */ -extern void copy(const vespalib::stringref & frompath, - const vespalib::stringref & topath, +extern void copy(const vespalib::string & frompath, + const vespalib::string & topath, bool createTargetDirectoryIfMissing = false, bool useDirectIO = true); @@ -438,11 +438,11 @@ extern void copy(const vespalib::stringref & frompath, * List the contents of the given directory. */ typedef std::vector<vespalib::string> DirectoryList; -extern DirectoryList listDirectory(const vespalib::stringref & path); +extern DirectoryList listDirectory(const vespalib::string & path); extern MallocAutoPtr getAlignedBuffer(size_t size); -string dirname(const stringref name); -string getOpenErrorString(const int osError, const stringref name); +string dirname(stringref name); +string getOpenErrorString(const int osError, stringref name); } // vespalib diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index b51fff1b7cc..c3127e06133 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -71,7 +71,7 @@ public: return *this; } nbostream & operator << (const char * v) { uint32_t sz(strlen(v)); (*this) << sz; write(v, sz); return *this; } - nbostream & operator << (const vespalib::stringref & v) { uint32_t sz(v.size()); (*this) << sz; write(v.c_str(), sz); return *this; } + nbostream & operator << (const vespalib::stringref & v) { uint32_t sz(v.size()); (*this) << sz; write(v.data(), sz); return *this; } nbostream & operator << (const vespalib::string & v) { uint32_t sz(v.size()); (*this) << sz; write(v.c_str(), sz); return *this; } nbostream & operator >> (vespalib::string & v) { uint32_t sz; (*this) >> sz; diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp index 1be24175ede..7e8570b3d61 100644 --- a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp +++ b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp @@ -520,7 +520,7 @@ void asciistream::write(const void * buf, size_t len) if (_rPos > 0 && _rPos == length()) { clear(); } - if (_rbuf.c_str() != _wbuf.c_str()) { + if (_rbuf.data() != _wbuf.data()) { if (_wbuf.empty()) { _wbuf = _rbuf; // Read only to RW } else { @@ -557,7 +557,7 @@ string asciistream::getline(char delim) asciistream asciistream::createFromFile(const stringref & fileName) { - FastOS_File file(fileName.c_str()); + FastOS_File file(vespalib::string(fileName).c_str()); asciistream is; if (file.OpenReadOnly()) { ssize_t sz = file.getSize(); @@ -578,7 +578,7 @@ asciistream asciistream::createFromFile(const stringref & fileName) asciistream asciistream::createFromDevice(const stringref & fileName) { - FastOS_File file(fileName.c_str()); + FastOS_File file(vespalib::string(fileName).c_str()); asciistream is; if (file.OpenReadOnly()) { char buf[8192]; diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.h b/vespalib/src/vespa/vespalib/stllike/asciistream.h index 1e2b4bc1823..9dd73706d0a 100644 --- a/vespalib/src/vespa/vespalib/stllike/asciistream.h +++ b/vespalib/src/vespa/vespalib/stllike/asciistream.h @@ -42,9 +42,9 @@ public: asciistream & operator << (char v) { doFill(1); write(&v, 1); return *this; } asciistream & operator << (unsigned char v) { doFill(1); write(&v, 1); return *this; } asciistream & operator << (const char * v) { if (v != nullptr) { size_t n(strlen(v)); doFill(n); write(v, n); } return *this; } - asciistream & operator << (const string & v) { doFill(v.size()); write(v.c_str(), v.size()); return *this; } - asciistream & operator << (const stringref & v) { doFill(v.size()); write(v.c_str(), v.size()); return *this; } - asciistream & operator << (const std::string & v) { doFill(v.size()); write(v.c_str(), v.size()); return *this; } + asciistream & operator << (const string & v) { doFill(v.size()); write(v.data(), v.size()); return *this; } + asciistream & operator << (const stringref & v) { doFill(v.size()); write(v.data(), v.size()); return *this; } + asciistream & operator << (const std::string & v) { doFill(v.size()); write(v.data(), v.size()); return *this; } asciistream & operator << (int16_t v) { return *this << static_cast<int64_t>(v); } asciistream & operator << (uint16_t v) { return *this << static_cast<uint64_t>(v); } asciistream & operator << (int32_t v) { return *this << static_cast<int64_t>(v); } @@ -74,7 +74,7 @@ public: asciistream & operator >> (float & v); asciistream & operator >> (double & v); stringref str() const { return stringref(c_str(), size()); } - const char * c_str() const { return _rbuf.c_str() + _rPos; } + const char * c_str() const { return _rbuf.data() + _rPos; } size_t size() const { return length() - _rPos; } bool empty() const { return size() == 0; } bool eof() const { return empty(); } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h index 8b14d5ce67c..7d7be666136 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h @@ -67,7 +67,7 @@ template<> struct hash<const char *> { }; template<> struct hash<vespalib::stringref> { - size_t operator() (const vespalib::stringref & arg) const { return hashValue(arg.c_str(), arg.size()); } + size_t operator() (const vespalib::stringref & arg) const { return hashValue(arg.data(), arg.size()); } }; template<> struct hash<vespalib::string> { diff --git a/vespalib/src/vespa/vespalib/stllike/string.cpp b/vespalib/src/vespa/vespalib/stllike/string.cpp index 081d45936c7..47d424e93aa 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.cpp +++ b/vespalib/src/vespa/vespalib/stllike/string.cpp @@ -47,7 +47,7 @@ stringref::find(const stringref & s, size_type start) const { std::ostream & operator << (std::ostream & os, const stringref & v) { - return os.write(v.c_str(), v.size()); + return os.write(v.data(), v.size()); } template<uint32_t SS> diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h index 3db36f5fd41..98ed0929a9b 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.h +++ b/vespalib/src/vespa/vespalib/stllike/string.h @@ -31,24 +31,21 @@ public: * return a pointer to the data held, or NULL. * Note that the data may not be zero terminated, and a default * constructed stringref will give a NULL pointer back. If you - * need to make sure c_str() gives a valid zero-terminated string + * need to make sure data() gives a valid zero-terminated string * you should make a vespalib::string from the stringref. **/ - const char * c_str() const { return _s; } - - /** return a pointer to the data held, or NULL. See c_str(). */ const char * data() const { return _s; } size_type size() const { return _sz; } size_type length() const { return size(); } bool empty() const { return _sz == 0; } - const char * begin() const { return c_str(); } + const char * begin() const { return data(); } const char * end() const { return begin() + size(); } const char * rbegin() const { return end() - 1; } const char * rend() const { return begin() - 1; } stringref substr(size_type start, size_type sz=npos) const { if (start < size()) { - return stringref(c_str() + start, std::min(sz, size()-start)); + return stringref(data() + start, std::min(sz, size()-start)); } return stringref(); } @@ -119,7 +116,7 @@ public: * was found, or npos if the substring could not be located */ size_type rfind(const char * s, size_type e=npos) const; - int compare(const stringref & s) const { return compare(s.c_str(), s.size()); } + int compare(const stringref & s) const { return compare(s.data(), s.size()); } int compare(const char *s, size_type sz) const { int diff(memcmp(_s, s, std::min(sz, size()))); return (diff != 0) ? diff : (size() - sz); @@ -127,23 +124,23 @@ public: const char & operator [] (size_t i) const { return _s[i]; } operator std::string () const { return std::string(_s, _sz); } bool operator < (const char * s) const { return compare(s, strlen(s)) < 0; } - bool operator < (const std::string & s) const { return compare(s.c_str(), s.size()) < 0; } - bool operator < (const stringref & s) const { return compare(s.c_str(), s.size()) < 0; } + bool operator < (const std::string & s) const { return compare(s.data(), s.size()) < 0; } + bool operator < (const stringref & s) const { return compare(s.data(), s.size()) < 0; } bool operator <= (const char * s) const { return compare(s, strlen(s)) <= 0; } - bool operator <= (const std::string & s) const { return compare(s.c_str(), s.size()) <= 0; } - bool operator <= (const stringref & s) const { return compare(s.c_str(), s.size()) <= 0; } + bool operator <= (const std::string & s) const { return compare(s.data(), s.size()) <= 0; } + bool operator <= (const stringref & s) const { return compare(s.data(), s.size()) <= 0; } bool operator != (const char * s) const { return compare(s, strlen(s)) != 0; } - bool operator != (const std::string & s) const { return compare(s.c_str(), s.size()) != 0; } - bool operator != (const stringref & s) const { return compare(s.c_str(), s.size()) != 0; } + bool operator != (const std::string & s) const { return compare(s.data(), s.size()) != 0; } + bool operator != (const stringref & s) const { return compare(s.data(), s.size()) != 0; } bool operator == (const char * s) const { return compare(s, strlen(s)) == 0; } - bool operator == (const std::string & s) const { return compare(s.c_str(), s.size()) == 0; } - bool operator == (const stringref & s) const { return compare(s.c_str(), s.size()) == 0; } + bool operator == (const std::string & s) const { return compare(s.data(), s.size()) == 0; } + bool operator == (const stringref & s) const { return compare(s.data(), s.size()) == 0; } bool operator >= (const char * s) const { return compare(s, strlen(s)) >= 0; } - bool operator >= (const std::string & s) const { return compare(s.c_str(), s.size()) >= 0; } - bool operator >= (const stringref & s) const { return compare(s.c_str(), s.size()) >= 0; } + bool operator >= (const std::string & s) const { return compare(s.data(), s.size()) >= 0; } + bool operator >= (const stringref & s) const { return compare(s.data(), s.size()) >= 0; } bool operator > (const char * s) const { return compare(s, strlen(s)) > 0; } - bool operator > (const std::string & s) const { return compare(s.c_str(), s.size()) > 0; } - bool operator > (const stringref & s) const { return compare(s.c_str(), s.size()) > 0; } + bool operator > (const std::string & s) const { return compare(s.data(), s.size()) > 0; } + bool operator > (const stringref & s) const { return compare(s.data(), s.size()) > 0; } private: const char *_s; size_type _sz; @@ -178,13 +175,13 @@ public: small_string() : _buf(_stack), _sz(0), _bufferSize(StackSize) { _stack[0] = '\0'; } small_string(const char * s) : _buf(_stack), _sz(s ? strlen(s) : 0) { init(s); } small_string(const void * s, size_type sz) : _buf(_stack), _sz(sz) { init(s); } - small_string(const stringref & s) : _buf(_stack), _sz(s.size()) { init(s.c_str()); } - small_string(const std::string & s) : _buf(_stack), _sz(s.size()) { init(s.c_str()); } - small_string(const small_string & rhs) noexcept : _buf(_stack), _sz(rhs.size()) { init(rhs.c_str()); } + small_string(const stringref & s) : _buf(_stack), _sz(s.size()) { init(s.data()); } + small_string(const std::string & s) : _buf(_stack), _sz(s.size()) { init(s.data()); } + small_string(const small_string & rhs) noexcept : _buf(_stack), _sz(rhs.size()) { init(rhs.data()); } small_string(const small_string & rhs, size_type pos, size_type sz=npos) noexcept : _buf(_stack), _sz(std::min(sz, rhs.size()-pos)) { - init(rhs.c_str()+pos); + init(rhs.data()+pos); } small_string(size_type sz, char c) : _buf(_stack), _sz(0), _bufferSize(StackSize) @@ -204,10 +201,10 @@ public: } } small_string& operator= (const small_string &rhs) { - return assign(rhs.c_str(), rhs.size()); + return assign(rhs.data(), rhs.size()); } small_string & operator= (const stringref &rhs) { - return assign(rhs.c_str(), rhs.size()); + return assign(rhs.data(), rhs.size()); } small_string& operator= (const char *s) { return assign(s); @@ -321,18 +318,18 @@ public: small_string & assign(const char * s) { return assign(s, strlen(s)); } small_string & assign(const void * s, size_type sz); small_string & assign(const stringref &s, size_type pos, size_type sz) { - return assign(s.c_str() + pos, sz); + return assign(s.data() + pos, sz); } small_string & assign(const stringref &rhs) { - if (c_str() != rhs.c_str()) assign(rhs.c_str(), rhs.size()); + if (data() != rhs.data()) assign(rhs.data(), rhs.size()); return *this; } small_string & push_back(char c) { return append(&c, 1); } small_string & append(char c) { return append(&c, 1); } small_string & append(const char * s) { return append(s, strlen(s)); } - small_string & append(const stringref & s) { return append(s.c_str(), s.size()); } - small_string & append(const std::string & s) { return append(s.c_str(), s.size()); } - small_string & append(const small_string & s) { return append(s.c_str(), s.size()); } + small_string & append(const stringref & s) { return append(s.data(), s.size()); } + small_string & append(const std::string & s) { return append(s.data(), s.size()); } + small_string & append(const small_string & s) { return append(s.data(), s.size()); } small_string & append(const void * s, size_type sz); small_string & operator += (char c) { return append(c); } small_string & operator += (const char * s) { return append(s); } @@ -358,7 +355,7 @@ public: } small_string & insert(iterator p, const_iterator f, const_iterator l) { return insert(p-c_str(), f, l-f); } - small_string & insert(size_type start, const stringref & v) { return insert(start, v.c_str(), v.size()); } + small_string & insert(size_type start, const stringref & v) { return insert(start, v.data(), v.size()); } small_string & insert(size_type start, const void * v, size_type sz); /** @@ -447,29 +444,29 @@ public: */ bool operator < (const char * s) const { return compare(s, strlen(s)) < 0; } - bool operator < (const std::string & s) const { return compare(s.c_str(), s.size()) < 0; } - bool operator < (const small_string & s) const { return compare(s.c_str(), s.size()) < 0; } - bool operator < (const stringref & s) const { return compare(s.c_str(), s.size()) < 0; } + bool operator < (const std::string & s) const { return compare(s.data(), s.size()) < 0; } + bool operator < (const small_string & s) const { return compare(s.data(), s.size()) < 0; } + bool operator < (const stringref & s) const { return compare(s.data(), s.size()) < 0; } bool operator <= (const char * s) const { return compare(s, strlen(s)) <= 0; } - bool operator <= (const std::string & s) const { return compare(s.c_str(), s.size()) <= 0; } - bool operator <= (const small_string & s) const { return compare(s.c_str(), s.size()) <= 0; } - bool operator <= (const stringref & s) const { return compare(s.c_str(), s.size()) <= 0; } + bool operator <= (const std::string & s) const { return compare(s.data(), s.size()) <= 0; } + bool operator <= (const small_string & s) const { return compare(s.data(), s.size()) <= 0; } + bool operator <= (const stringref & s) const { return compare(s.data(), s.size()) <= 0; } bool operator == (const char * s) const { return compare(s, strlen(s)) == 0; } - bool operator == (const std::string & s) const { return compare(s.c_str(), s.size()) == 0; } - bool operator == (const small_string & s) const { return compare(s.c_str(), s.size()) == 0; } - bool operator == (const stringref & s) const { return compare(s.c_str(), s.size()) == 0; } + bool operator == (const std::string & s) const { return compare(s.data(), s.size()) == 0; } + bool operator == (const small_string & s) const { return compare(s.data(), s.size()) == 0; } + bool operator == (const stringref & s) const { return compare(s.data(), s.size()) == 0; } bool operator != (const char * s) const { return compare(s, strlen(s)) != 0; } - bool operator != (const std::string & s) const { return compare(s.c_str(), s.size()) != 0; } - bool operator != (const small_string & s) const { return compare(s.c_str(), s.size()) != 0; } - bool operator != (const stringref & s) const { return compare(s.c_str(), s.size()) != 0; } + bool operator != (const std::string & s) const { return compare(s.data(), s.size()) != 0; } + bool operator != (const small_string & s) const { return compare(s.data(), s.size()) != 0; } + bool operator != (const stringref & s) const { return compare(s.data(), s.size()) != 0; } bool operator >= (const char * s) const { return compare(s, strlen(s)) >= 0; } - bool operator >= (const std::string & s) const { return compare(s.c_str(), s.size()) >= 0; } - bool operator >= (const small_string & s) const { return compare(s.c_str(), s.size()) >= 0; } - bool operator >= (const stringref & s) const { return compare(s.c_str(), s.size()) >= 0; } + bool operator >= (const std::string & s) const { return compare(s.data(), s.size()) >= 0; } + bool operator >= (const small_string & s) const { return compare(s.data(), s.size()) >= 0; } + bool operator >= (const stringref & s) const { return compare(s.data(), s.size()) >= 0; } bool operator > (const char * s) const { return compare(s, strlen(s)) > 0; } - bool operator > (const std::string & s) const { return compare(s.c_str(), s.size()) > 0; } - bool operator > (const small_string & s) const { return compare(s.c_str(), s.size()) > 0; } - bool operator > (const stringref & s) const { return compare(s.c_str(), s.size()) > 0; } + bool operator > (const std::string & s) const { return compare(s.data(), s.size()) > 0; } + bool operator > (const small_string & s) const { return compare(s.data(), s.size()) > 0; } + bool operator > (const stringref & s) const { return compare(s.data(), s.size()) > 0; } template<typename T> bool operator != (const T& s) const { return ! operator == (s); } diff --git a/vespalib/src/vespa/vespalib/util/exceptions.cpp b/vespalib/src/vespa/vespalib/util/exceptions.cpp index 3db697e3b68..fc64aa79ca4 100644 --- a/vespalib/src/vespa/vespalib/util/exceptions.cpp +++ b/vespalib/src/vespa/vespalib/util/exceptions.cpp @@ -74,7 +74,8 @@ PortListenException::make_message(int port, const vespalib::stringref &protocol, const vespalib::stringref &msg) { return make_string("failed to listen on port %d with protocol %s%s%s", - port, protocol.c_str(), msg.empty() ? "" : ": ", msg.c_str()); + port, vespalib::string(protocol).c_str(), msg.empty() ? "" : ": ", + vespalib::string(msg).c_str()); } PortListenException::PortListenException(int port, const vespalib::stringref &protocol, diff --git a/vespalib/src/vespa/vespalib/util/regexp.cpp b/vespalib/src/vespa/vespalib/util/regexp.cpp index b376b451b6d..e1aa4e9b189 100644 --- a/vespalib/src/vespa/vespalib/util/regexp.cpp +++ b/vespalib/src/vespa/vespalib/util/regexp.cpp @@ -32,13 +32,13 @@ Regexp::compile(const vespalib::stringref & re, Flags flags) preg->fastmap = static_cast<char *>(malloc(256)); preg->buffer = NULL; preg->allocated = 0; - const char * error = re_compile_pattern(re.c_str(), re.size(), preg); + const char * error = re_compile_pattern(re.data(), re.size(), preg); if (error != 0) { - LOG(warning, "invalid regexp '%s': %s", re.c_str(), error); + LOG(warning, "invalid regexp '%s': %s", vespalib::string(re).c_str(), error); return false; } if (re_compile_fastmap(preg) != 0) { - LOG(warning, "re_compile_fastmap failed for regexp '%s'", re.c_str()); + LOG(warning, "re_compile_fastmap failed for regexp '%s'", vespalib::string(re).c_str()); return false; } return true; @@ -57,7 +57,7 @@ Regexp::match(const vespalib::stringref & s) const { if ( ! valid() ) { return false; } regex_t *preg = const_cast<regex_t *>(static_cast<const regex_t *>(_data)); - int pos(re_search(preg, s.c_str(), s.size(), 0, s.size(), NULL)); + int pos(re_search(preg, s.data(), s.size(), 0, s.size(), NULL)); if (pos < -1) { throw IllegalArgumentException(make_string("re_search failed with code(%d)", pos)); } @@ -70,13 +70,13 @@ vespalib::string Regexp::replace(const vespalib::stringref & s, const vespalib:: regex_t *preg = const_cast<regex_t *>(static_cast<const regex_t *>(_data)); vespalib::string modified; int prev(0); - for(int pos(re_search(preg, s.c_str(), s.size(), 0, s.size(), NULL)); + for(int pos(re_search(preg, s.data(), s.size(), 0, s.size(), NULL)); pos >=0; - pos = re_search(preg, s.c_str()+prev, s.size()-prev, 0, s.size()-prev, NULL)) + pos = re_search(preg, s.data()+prev, s.size()-prev, 0, s.size()-prev, NULL)) { modified += s.substr(prev, pos); modified += replacement; - int count = re_match(preg, s.c_str()+prev, s.size()-prev, pos, NULL); + int count = re_match(preg, s.data()+prev, s.size()-prev, pos, NULL); prev += pos + count; } modified += s.substr(prev); diff --git a/vespalib/src/vespa/vespalib/util/rendezvous.h b/vespalib/src/vespa/vespalib/util/rendezvous.h index 383793ce2a3..5370968c408 100644 --- a/vespalib/src/vespa/vespalib/util/rendezvous.h +++ b/vespalib/src/vespa/vespalib/util/rendezvous.h @@ -71,7 +71,7 @@ public: * @param n the size of this Rendezvous **/ Rendezvous(size_t n); - virtual ~Rendezvous() {} + virtual ~Rendezvous(); /** * Called by individual threads to synchronize execution and share diff --git a/vespalib/src/vespa/vespalib/util/rendezvous.hpp b/vespalib/src/vespa/vespalib/util/rendezvous.hpp index 423e33c6080..07112055de7 100644 --- a/vespalib/src/vespa/vespalib/util/rendezvous.hpp +++ b/vespalib/src/vespa/vespalib/util/rendezvous.hpp @@ -19,6 +19,9 @@ Rendezvous<IN, OUT>::Rendezvous(size_t n) } template <typename IN, typename OUT> +Rendezvous<IN, OUT>::~Rendezvous() = default; + +template <typename IN, typename OUT> OUT Rendezvous<IN, OUT>::rendezvous(const IN &input) { diff --git a/vsm/src/vespa/vsm/common/document.cpp b/vsm/src/vespa/vsm/common/document.cpp index dc820cd99b3..7e4713e672c 100644 --- a/vsm/src/vespa/vsm/common/document.cpp +++ b/vsm/src/vespa/vsm/common/document.cpp @@ -12,10 +12,10 @@ namespace vsm vespalib::asciistream & operator << (vespalib::asciistream & os, const FieldRef & f) { - const char *s = f.c_str(); + const char *s = f.data(); os << f.size(); if (s) { - os << s; + os << s; // Better hope it's null terminated! } os << " : "; return os; diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp index 78531f41cf8..2ba5fea3153 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp @@ -115,7 +115,7 @@ void FieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & UNUSE size_t FieldSearcher::countWords(const FieldRef & f) { size_t words = 0; - const char * n = f.c_str(); + const char * n = f.data(); const char * e = n + f.size(); for( ; n < e; ++n) { for (; isspace(*n) && (n<e); ++n); diff --git a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp index b388507aed5..b26b4bd5133 100644 --- a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp @@ -222,7 +222,7 @@ size_t FUTF8StrChrFieldSearcher::matchTerm(const FieldRef & f, QueryTerm & qt) { _folded.reserve(f.size()+16*3); //Enable fulle xmm0 store size_t unalignedStart(0); - bool ascii7Bit = lfoldua(f.c_str(), f.size(), &_folded[0], unalignedStart); + bool ascii7Bit = lfoldua(f.data(), f.size(), &_folded[0], unalignedStart); if (ascii7Bit) { char * folded = &_folded[unalignedStart]; /// Add the pattern 00 01 00 to avoid multiple eof tests of falling off the edge. @@ -240,7 +240,7 @@ size_t FUTF8StrChrFieldSearcher::matchTerms(const FieldRef & f, const size_t min { _folded.reserve(f.size()+16*3); //Enable fulle xmm0 store size_t unalignedStart(0); - bool ascii7Bit = lfoldua(f.c_str(), f.size(), &_folded[0], unalignedStart); + bool ascii7Bit = lfoldua(f.data(), f.size(), &_folded[0], unalignedStart); if (ascii7Bit) { char * folded = &_folded[unalignedStart]; /// Add the pattern 00 01 00 to avoid multiple eof tests of falling off the edge. diff --git a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp index 93bfa76081f..1be1326807e 100644 --- a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp @@ -16,7 +16,7 @@ void StrChrFieldSearcher::onValue(const document::FieldValue & fv) { const document::LiteralFieldValueB & sfv = static_cast<const document::LiteralFieldValueB &>(fv); vespalib::stringref val = sfv.getValueRef(); - FieldRef fr(val.c_str(), std::min(maxFieldLength(), val.size())); + FieldRef fr(val.data(), std::min(maxFieldLength(), val.size())); matchDoc(fr); } diff --git a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp index 82821dfeaff..b54ed2c583d 100644 --- a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp @@ -14,7 +14,7 @@ UTF8StrChrFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz) { (void) mintsz; termcount_t words(0); - const byte * n = reinterpret_cast<const byte *> (f.c_str()); + const byte * n = reinterpret_cast<const byte *> (f.data()); const byte * e = n + f.size(); if (f.size() >= _buf->size()) { _buf->reserve(f.size() + 1); diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp index 0ce002e6765..872cfebfd70 100644 --- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp @@ -104,7 +104,7 @@ size_t UTF8StringFieldSearcherBase::matchTermRegular(const FieldRef & f, QueryTerm & qt) { termcount_t words(0); - const byte * n = reinterpret_cast<const byte *> (f.c_str()); + const byte * n = reinterpret_cast<const byte *> (f.data()); // __builtin_prefetch(n, 0, 0); const cmptype_t * term; termsize_t tsz = qt.term(term); @@ -134,7 +134,7 @@ UTF8StringFieldSearcherBase::matchTermRegular(const FieldRef & f, QueryTerm & qt size_t UTF8StringFieldSearcherBase::matchTermExact(const FieldRef & f, QueryTerm & qt) { - const byte * n = reinterpret_cast<const byte *> (f.c_str()); + const byte * n = reinterpret_cast<const byte *> (f.data()); const cmptype_t * term; termsize_t tsz = qt.term(term); const cmptype_t * eterm = term+tsz; @@ -161,7 +161,7 @@ size_t UTF8StringFieldSearcherBase::matchTermSubstring(const FieldRef & f, QueryTerm & qt) { if (qt.termLen() == 0) { return 0; } - const byte * n = reinterpret_cast<const byte *> (f.c_str()); + const byte * n = reinterpret_cast<const byte *> (f.data()); const cmptype_t * term; termsize_t tsz = qt.term(term); if ( f.size() >= _buf->size()) { @@ -195,7 +195,7 @@ size_t UTF8StringFieldSearcherBase::matchTermSuffix(const FieldRef & f, QueryTerm & qt) { termcount_t words = 0; - const byte * srcbuf = reinterpret_cast<const byte *> (f.c_str()); + const byte * srcbuf = reinterpret_cast<const byte *> (f.data()); const byte * srcend = srcbuf + f.size(); const cmptype_t * term; termsize_t tsz = qt.term(term); diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp index 94fb14b6217..4b8c6e31927 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp @@ -13,7 +13,7 @@ IMPLEMENT_DUPLICATE(UTF8SubStringFieldSearcher); size_t UTF8SubStringFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz) { - const byte * n = reinterpret_cast<const byte *> (f.c_str()); + const byte * n = reinterpret_cast<const byte *> (f.data()); if ( f.size() >= _buf->size()) { _buf->reserve(f.size() + 1); } diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp index b229b7a5ebd..eee88b34ea6 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp @@ -14,8 +14,8 @@ size_t UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz) { _modified->reset(); - _readPtr = f.c_str(); - const byte * src = reinterpret_cast<const byte *> (f.c_str()); + _readPtr = f.data(); + const byte * src = reinterpret_cast<const byte *> (f.data()); // resize ucs4 buffer if (f.size() >= _buf->size()) { _buf->resize(f.size() + 1); @@ -46,8 +46,8 @@ UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz const cmptype_t * dtmp = ditr; for (; (titr < tend) && (*titr == *dtmp); ++titr, ++dtmp); if (titr == tend) { - const char * mbegin = f.c_str() + (*_offsets)[ditr - dbegin]; - const char * mend = f.c_str() + ((dtmp < dend) ? ((*_offsets)[dtmp - dbegin]) : f.size()); + const char * mbegin = f.data() + (*_offsets)[ditr - dbegin]; + const char * mend = f.data() + ((dtmp < dend) ? ((*_offsets)[dtmp - dbegin]) : f.size()); if (_readPtr <= mbegin) { // We will only copy from the field ref once. // If we have overlapping matches only the first one will be considered. @@ -61,9 +61,9 @@ UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz for(; (ditr < drend) && ! Fast_UnicodeUtil::IsWordChar(*ditr) ; ++ditr ); } } - assert(_readPtr <= (f.c_str() + f.size())); + assert(_readPtr <= (f.data() + f.size())); // copy remaining - size_t toCopy = f.size() - (_readPtr - f.c_str()); + size_t toCopy = f.size() - (_readPtr - f.data()); copyToModified(toCopy); return words + 1; // we must also count the last word diff --git a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp index f469fa2ea73..13074937185 100644 --- a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp +++ b/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp @@ -14,7 +14,7 @@ UTF8SuffixStringFieldSearcher::matchTerms(const FieldRef & f, const size_t mints { (void) mintsz; termcount_t words = 0; - const byte * srcbuf = reinterpret_cast<const byte *> (f.c_str()); + const byte * srcbuf = reinterpret_cast<const byte *> (f.data()); const byte * srcend = srcbuf + f.size(); if (f.size() >= _buf->size()) { _buf->reserve(f.size() + 1); diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp index e6af5fb6477..034c3c57bde 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp @@ -74,7 +74,7 @@ public: if (fv.getClass().inherits(document::LiteralFieldValueB::classId)) { const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv); vespalib::stringref s = lfv.getValueRef(); - addToPacker(s.c_str(), s.size()); + addToPacker(s.data(), s.size()); } else { vespalib::string s = fv.toString(); addToPacker(s.c_str(), s.size()); diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp index bf690e50719..080723e1dbd 100644 --- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp +++ b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp @@ -21,13 +21,13 @@ FlattenDocsumWriter::onPrimitive(uint32_t, const Content & c) if (fv.getClass().inherits(document::LiteralFieldValueB::classId)) { const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv); vespalib::stringref value = lfv.getValueRef(); - _output.put(value.c_str(), value.size()); + _output.put(value.data(), value.size()); } else if (fv.getClass().inherits(document::NumericFieldValueBase::classId)) { vespalib::string value = fv.getAsString(); - _output.put(value.c_str(), value.size()); + _output.put(value.data(), value.size()); } else { vespalib::string value = fv.toString(); - _output.put(value.c_str(), value.size()); + _output.put(value.data(), value.size()); } _useSeparator = true; } |