diff options
138 files changed, 1968 insertions, 1559 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 89e7e340641..f1a93e58526 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -35,15 +34,12 @@ import java.util.stream.Stream; */ public class InstanceValidator { + private static final AthenzService TENANT_DOCKER_CONTAINER_IDENTITY = new AthenzService("vespa.vespa.tenant"); private static final Logger log = Logger.getLogger(InstanceValidator.class.getName()); static final String SERVICE_PROPERTIES_DOMAIN_KEY = "identity.domain"; static final String SERVICE_PROPERTIES_SERVICE_KEY = "identity.service"; static final String INSTANCE_ID_DELIMITER = ".instanceid.athenz."; - public static final String SAN_IPS_ATTRNAME = "sanIP"; - public static final String SAN_DNS_ATTRNAME = "sanDNS"; - - private final AthenzService tenantDockerContainerIdentity; private final IdentityDocumentSigner signer; private final KeyProvider keyProvider; private final SuperModelProvider superModelProvider; @@ -52,21 +48,18 @@ public class InstanceValidator { @Inject public InstanceValidator(KeyProvider keyProvider, SuperModelProvider superModelProvider, - NodeRepository nodeRepository, - AthenzProviderServiceConfig config) { - this(keyProvider, superModelProvider, nodeRepository, new IdentityDocumentSigner(), new AthenzService(config.tenantService())); + NodeRepository nodeRepository) { + this(keyProvider, superModelProvider, nodeRepository, new IdentityDocumentSigner()); } public InstanceValidator(KeyProvider keyProvider, SuperModelProvider superModelProvider, NodeRepository nodeRepository, - IdentityDocumentSigner identityDocumentSigner, - AthenzService tenantIdentity){ + IdentityDocumentSigner identityDocumentSigner){ this.keyProvider = keyProvider; this.superModelProvider = superModelProvider; this.nodeRepository = nodeRepository; this.signer = identityDocumentSigner; - this.tenantDockerContainerIdentity = tenantIdentity; } public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { @@ -103,7 +96,7 @@ public class InstanceValidator { log.log(LogLevel.INFO, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", new AthenzService(confirmation.domain, confirmation.service).getFullName(), confirmation.provider, - confirmation.attributes.get(SAN_DNS_ATTRNAME))); + confirmation.attributes.get("sanDNS"))); try { return validateAttributes(confirmation, getVespaUniqueInstanceId(confirmation)); } catch (Exception e) { @@ -114,7 +107,7 @@ public class InstanceValidator { private VespaUniqueInstanceId getVespaUniqueInstanceId(InstanceConfirmation instanceConfirmation) { // Find a list of SAN DNS - List<String> sanDNS = Optional.ofNullable(instanceConfirmation.attributes.get(SAN_DNS_ATTRNAME)) + List<String> sanDNS = Optional.ofNullable(instanceConfirmation.attributes.get("sanDNS")) .map(s -> s.split(",")) .map(Arrays::asList) .map(List::stream) @@ -131,7 +124,7 @@ public class InstanceValidator { private boolean validateAttributes(InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) { if(vespaUniqueInstanceId == null) { - log.log(LogLevel.WARNING, "Unable to find unique instance ID in refresh request: " + confirmation.toString()); + log.log(LogLevel.WARNING, "Unabe to find unique instance ID in refresh request: " + confirmation.toString()); return false; } @@ -147,7 +140,7 @@ public class InstanceValidator { } // Find list of ipaddresses - List<InetAddress> ips = Optional.ofNullable(confirmation.attributes.get(SAN_IPS_ATTRNAME)) + List<InetAddress> ips = Optional.ofNullable(confirmation.attributes.get("sanIP")) .map(s -> s.split(",")) .map(Arrays::asList) .map(List::stream) @@ -191,7 +184,7 @@ public class InstanceValidator { return false; } - if (tenantDockerContainerIdentity.equals(new AthenzService(domain, service))) { + if (TENANT_DOCKER_CONTAINER_IDENTITY.equals(new AthenzService(domain, service))) { return true; } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java index b5a2405167a..308127e29c7 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/Certificates.java @@ -4,17 +4,14 @@ package com.yahoo.vespa.hosted.ca; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; -import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static com.yahoo.security.SubjectAlternativeName.Type.DNS_NAME; @@ -56,22 +53,14 @@ public class Certificates { /** Returns instance ID parsed from the Subject Alternative Names in given csr */ public static String instanceIdFrom(Pkcs10Csr csr) { - return getInstanceIdFromSAN(csr.getSubjectAlternativeNames()) - .orElseThrow(() -> new IllegalArgumentException("No instance ID found in CSR")); - } - - public static Optional<String> instanceIdFrom(X509Certificate certificate) { - return getInstanceIdFromSAN(X509CertificateUtils.getSubjectAlternativeNames(certificate)); - } - - private static Optional<String> getInstanceIdFromSAN(List<SubjectAlternativeName> subjectAlternativeNames) { - return subjectAlternativeNames.stream() - .filter(san -> san.getType() == DNS_NAME) - .map(SubjectAlternativeName::getValue) - .map(Certificates::parseInstanceId) - .flatMap(Optional::stream) - .map(VespaUniqueInstanceId::asDottedString) - .findFirst(); + return csr.getSubjectAlternativeNames().stream() + .filter(san -> san.getType() == DNS_NAME) + .map(SubjectAlternativeName::getValue) + .map(Certificates::parseInstanceId) + .flatMap(Optional::stream) + .map(VespaUniqueInstanceId::asDottedString) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No instance ID found in CSR")); } private static Optional<VespaUniqueInstanceId> parseInstanceId(String dnsName) { @@ -85,11 +74,4 @@ public class Certificates { } } - public static String getSubjectAlternativeNames(Pkcs10Csr csr, SubjectAlternativeName.Type sanType) { - return csr.getSubjectAlternativeNames().stream() - .map(SubjectAlternativeName::decode) - .filter(san -> san.getType() == sanType) - .map(SubjectAlternativeName::getValue) - .collect(Collectors.joining(",")); - } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java index 8d9f44e4564..2a2b702d21b 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/instance/InstanceRegistration.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.ca.instance; import com.yahoo.security.Pkcs10Csr; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import java.util.Objects; @@ -17,10 +16,10 @@ public class InstanceRegistration { private final String provider; private final String domain; private final String service; - private final SignedIdentityDocument attestationData; + private final String attestationData; private final Pkcs10Csr csr; - public InstanceRegistration(String provider, String domain, String service, SignedIdentityDocument attestationData, Pkcs10Csr csr) { + public InstanceRegistration(String provider, String domain, String service, String attestationData, Pkcs10Csr csr) { this.provider = Objects.requireNonNull(provider, "provider must be non-null"); this.domain = Objects.requireNonNull(domain, "domain must be non-null"); this.service = Objects.requireNonNull(service, "service must be non-null"); @@ -44,7 +43,7 @@ public class InstanceRegistration { } /** Host document describing this instance (received from config server) */ - public SignedIdentityDocument attestationData() { + public String attestationData() { return attestationData; } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 4c01b0943e4..1b9bdcdb987 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -6,24 +6,16 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.servlet.ServletRequest; -import com.yahoo.log.LogLevel; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.security.KeyUtils; -import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; import com.yahoo.vespa.hosted.ca.Certificates; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; -import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; import com.yahoo.yolean.Exceptions; import java.io.IOException; @@ -31,10 +23,6 @@ import java.io.UncheckedIOException; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.logging.Level; @@ -55,20 +43,18 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { private final Certificates certificates; private final String caPrivateKeySecretName; private final String caCertificateSecretName; - private final InstanceValidator instanceValidator; @Inject - public CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, AthenzProviderServiceConfig athenzProviderServiceConfig, InstanceValidator instanceValidator) { - this(ctx, secretStore, new Certificates(Clock.systemUTC()), athenzProviderServiceConfig, instanceValidator); + public CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, AthenzProviderServiceConfig athenzProviderServiceConfig) { + this(ctx, secretStore, new Certificates(Clock.systemUTC()), athenzProviderServiceConfig); } - CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, Certificates certificates, AthenzProviderServiceConfig athenzProviderServiceConfig, InstanceValidator instanceValidator) { + CertificateAuthorityApiHandler(Context ctx, SecretStore secretStore, Certificates certificates, AthenzProviderServiceConfig athenzProviderServiceConfig) { super(ctx); this.secretStore = secretStore; this.certificates = certificates; this.caPrivateKeySecretName = athenzProviderServiceConfig.secretName(); this.caCertificateSecretName = athenzProviderServiceConfig.domain() + ".ca.cert"; - this.instanceValidator = instanceValidator; } @Override @@ -95,14 +81,6 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { private HttpResponse registerInstance(HttpRequest request) { var instanceRegistration = deserializeRequest(request, InstanceSerializer::registrationFromSlime); - - InstanceConfirmation confirmation = new InstanceConfirmation(instanceRegistration.provider(), instanceRegistration.domain(), instanceRegistration.service(), EntityBindingsMapper.toSignedIdentityDocumentEntity(instanceRegistration.attestationData())); - confirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.IP_ADDRESS)); - confirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.DNS_NAME)); - if (!instanceValidator.isValidInstance(confirmation)) { - log.log(LogLevel.INFO, "Invalid instance registration for " + instanceRegistration.toString()); - return ErrorResponse.forbidden("Unable to launch service: " +instanceRegistration.service()); - } var certificate = certificates.create(instanceRegistration.csr(), caCertificate(), caPrivateKey()); var instanceId = Certificates.instanceIdFrom(instanceRegistration.csr()); var identity = new InstanceIdentity(instanceRegistration.provider(), instanceRegistration.service(), instanceId, @@ -113,65 +91,21 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { private HttpResponse refreshInstance(HttpRequest request, String provider, String service, String instanceId) { var instanceRefresh = deserializeRequest(request, InstanceSerializer::refreshFromSlime); var instanceIdFromCsr = Certificates.instanceIdFrom(instanceRefresh.csr()); - var athenzService = new AthenzService(request.getJDiscRequest().getUserPrincipal().getName()); if (!instanceIdFromCsr.equals(instanceId)) { throw new IllegalArgumentException("Mismatch between instance ID in URL path and instance ID in CSR " + "[instanceId=" + instanceId + ",instanceIdFromCsr=" + instanceIdFromCsr + "]"); } - - // Verify that the csr instance id matches one of the certificates in the chain - refreshesSameInstanceId(instanceIdFromCsr, request); - - - // Validate that there is no privilege escalation (can only refresh same service) - refreshesSameService(instanceRefresh, athenzService); - - InstanceConfirmation instanceConfirmation = new InstanceConfirmation(provider, athenzService.getDomain().getName(), athenzService.getName(), null); - instanceConfirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRefresh.csr(), SubjectAlternativeName.Type.IP_ADDRESS)); - instanceConfirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRefresh.csr(), SubjectAlternativeName.Type.DNS_NAME)); - if(!instanceValidator.isValidRefresh(instanceConfirmation)) { - return ErrorResponse.forbidden("Unable to refresh cert: " + instanceRefresh.csr().getSubject().toString()); - } - var certificate = certificates.create(instanceRefresh.csr(), caCertificate(), caPrivateKey()); var identity = new InstanceIdentity(provider, service, instanceIdFromCsr, Optional.of(certificate)); return new SlimeJsonResponse(InstanceSerializer.identityToSlime(identity)); } - public void refreshesSameInstanceId(String csrInstanceId, HttpRequest request) { - String certificateInstanceId = getRequestCertificateChain(request).stream() - .map(Certificates::instanceIdFrom) - .filter(Optional::isPresent) - .map(Optional::get) - .findAny().orElseThrow(() -> new IllegalArgumentException("No client certificate with instance id in request.")); - - if(! Objects.equals(certificateInstanceId, csrInstanceId)) { - throw new IllegalArgumentException("Mismatch between instance ID in client certificate and instance ID in CSR " + - "[instanceId=" + certificateInstanceId + ",instanceIdFromCsr=" + csrInstanceId + - "]"); - } - } - - private void refreshesSameService(InstanceRefresh instanceRefresh, AthenzService athenzService) { - List<String> commonNames = X509CertificateUtils.getCommonNames(instanceRefresh.csr().getSubject()); - if(commonNames.size() != 1 && !Objects.equals(commonNames.get(0), athenzService.getFullName())) { - throw new IllegalArgumentException(String.format("Invalid request, trying to refresh service %s using service %s.", instanceRefresh.csr().getSubject().getName(), athenzService.getFullName())); - } - } - /** Returns CA certificate from secret store */ private X509Certificate caCertificate() { return X509CertificateUtils.fromPem(secretStore.getSecret(caCertificateSecretName)); } - private List<X509Certificate> getRequestCertificateChain(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().context().get(ServletRequest.JDISC_REQUEST_X509CERT)) - .map(X509Certificate[].class::cast) - .map(Arrays::asList) - .orElse(Collections.emptyList()); - } - /** Returns CA private key from secret store */ private PrivateKey caPrivateKey() { return KeyUtils.fromPemEncodedPrivateKey(secretStore.getSecret(caPrivateKeySecretName)); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java index f08e5e37417..a2537cd68f1 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java @@ -3,23 +3,12 @@ package com.yahoo.vespa.hosted.ca.restapi; import com.yahoo.security.Pkcs10CsrUtils; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration; -import java.time.Instant; -import java.util.HashSet; -import java.util.Set; - /** * @author mpolden */ @@ -34,17 +23,6 @@ public class InstanceSerializer { private static final String INSTANCE_ID_FIELD = "instanceId"; private static final String X509_CERTIFICATE_FIELD = "x509Certificate"; - private static final String IDD_SIGNATURE_FIELD = "signature"; - private static final String IDD_SIGNING_KEY_VERSION_FIELD = "signing-key-version"; - private static final String IDD_PROVIDER_UNIQUE_ID_FIELD = "provider-unique-id"; - private static final String IDD_PROVIDER_SERVICE_FIELD = "provider-service"; - private static final String IDD_DOCUMENT_VERSION_FIELD = "document-version"; - private static final String IDD_CONFIGSERVER_HOSTNAME_FIELD = "configserver-hostname"; - private static final String IDD_INSTANCE_HOSTNAME_FIELD = "instance-hostname"; - private static final String IDD_CREATED_AT_FIELD = "created-at"; - private static final String IDD_IPADDRESSES_FIELD = "ip-addresses"; - private static final String IDD_IDENTITY_TYPE_FIELD = "identity-type"; - private InstanceSerializer() {} public static InstanceRegistration registrationFromSlime(Slime slime) { @@ -52,7 +30,7 @@ public class InstanceSerializer { return new InstanceRegistration(requireField(PROVIDER_FIELD, root).asString(), requireField(DOMAIN_FIELD, root).asString(), requireField(SERVICE_FIELD, root).asString(), - attestationDataToIdentityDocument(StringUtilities.unescape(requireField(ATTESTATION_DATA_FIELD, root).asString())), + requireField(ATTESTATION_DATA_FIELD, root).asString(), Pkcs10CsrUtils.fromPem(requireField(CSR_FIELD, root).asString())); } @@ -73,26 +51,6 @@ public class InstanceSerializer { return slime; } - private static SignedIdentityDocument attestationDataToIdentityDocument(String attestationData) { - Slime slime = SlimeUtils.jsonToSlime(attestationData); - Cursor root = slime.get(); - String signature = requireField(IDD_SIGNATURE_FIELD, root).asString(); - long signingKeyVersion = requireField(IDD_SIGNING_KEY_VERSION_FIELD, root).asLong(); - VespaUniqueInstanceId providerUniqueId = VespaUniqueInstanceId.fromDottedString(requireField(IDD_PROVIDER_UNIQUE_ID_FIELD, root).asString()); - AthenzService athenzService = new AthenzService(requireField(IDD_PROVIDER_SERVICE_FIELD, root).asString()); - long documentVersion = requireField(IDD_DOCUMENT_VERSION_FIELD, root).asLong(); - String configserverHostname = requireField(IDD_CONFIGSERVER_HOSTNAME_FIELD, root).asString(); - String instanceHostname = requireField(IDD_INSTANCE_HOSTNAME_FIELD, root).asString(); - double createdAtTimestamp = requireField(IDD_CREATED_AT_FIELD, root).asDouble(); - Instant createdAt = Instant.ofEpochSecond(Double.valueOf(createdAtTimestamp).longValue()); - Set<String> ips = new HashSet<>(); - requireField(IDD_IPADDRESSES_FIELD, root).traverse((ArrayTraverser) (__, entry) -> ips.add(entry.asString())); - IdentityType identityType = IdentityType.fromId(requireField(IDD_IDENTITY_TYPE_FIELD, root).asString()); - - return new SignedIdentityDocument(signature, (int)signingKeyVersion, providerUniqueId, athenzService, (int)documentVersion, - configserverHostname, instanceHostname, createdAt, ips, identityType); - } - private static Cursor requireField(String fieldName, Cursor root) { var field = root.field(fieldName); if (!field.valid()) throw new IllegalArgumentException("Missing required field '" + fieldName + "'"); diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java index a35dfd878c5..5ce0f3cdd7e 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java @@ -58,12 +58,10 @@ public class InstanceValidatorTest { private final String domain = "domain"; private final String service = "service"; - private final AthenzService vespaTenantDomain = new AthenzService("vespa.vespa.tenant"); - @Test public void application_does_not_exist() { SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null); assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -71,7 +69,7 @@ public class InstanceValidatorTest { public void application_does_not_have_domain_set() { SuperModelProvider superModelProvider = mockSuperModelProvider( mockApplicationInfo(applicationId, 5, Collections.emptyList())); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, new IdentityDocumentSigner(), vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null); assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -83,7 +81,7 @@ public class InstanceValidatorTest { SuperModelProvider superModelProvider = mockSuperModelProvider( mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null); assertFalse(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -101,7 +99,7 @@ public class InstanceValidatorTest { mockApplicationInfo(applicationId, 5, Collections.singletonList(serviceInfo))); IdentityDocumentSigner signer = mock(IdentityDocumentSigner.class); when(signer.hasValidSignature(any(), any())).thenReturn(true); - InstanceValidator instanceValidator = new InstanceValidator(mock(KeyProvider.class), superModelProvider, null, signer, vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(mock(KeyProvider.class), superModelProvider, null, signer); assertTrue(instanceValidator.isValidInstance(createRegisterInstanceConfirmation(applicationId, domain, service))); } @@ -109,7 +107,7 @@ public class InstanceValidatorTest { @Test public void rejects_invalid_provider_unique_id_in_csr() { SuperModelProvider superModelProvider = mockSuperModelProvider(); - InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null, vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, superModelProvider, null, null); InstanceConfirmation instanceConfirmation = createRegisterInstanceConfirmation(applicationId, domain, service); VespaUniqueInstanceId tamperedId = new VespaUniqueInstanceId(0, "default", "instance", "app", "tenant", "us-north-1", "dev", IdentityType.NODE); instanceConfirmation.set("sanDNS", tamperedId.asDottedString() + ".instanceid.athenz.dev-us-north-1.vespa.yahoo.cloud"); @@ -119,7 +117,7 @@ public class InstanceValidatorTest { @Test public void accepts_valid_refresh_requests() { NodeRepository nodeRepository = mock(NodeRepository.class); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository); List<Node> nodeList = createNodes(10); Node node = nodeList.get(0); @@ -134,7 +132,7 @@ public class InstanceValidatorTest { @Test public void rejects_refresh_on_ip_mismatch() { NodeRepository nodeRepository = mock(NodeRepository.class); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository); List<Node> nodeList = createNodes(10); Node node = nodeList.get(0); @@ -151,7 +149,7 @@ public class InstanceValidatorTest { @Test public void rejects_refresh_when_node_is_not_allocated() { NodeRepository nodeRepository = mock(NodeRepository.class); - InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository, new IdentityDocumentSigner(), vespaTenantDomain); + InstanceValidator instanceValidator = new InstanceValidator(null, null, nodeRepository); List<Node> nodeList = createNodes(10); when(nodeRepository.getNodes()).thenReturn(nodeList); diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java index 88ee154dee8..e377009b18c 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java @@ -2,34 +2,25 @@ package com.yahoo.vespa.hosted.ca.restapi; import com.yahoo.application.container.handler.Request; -import com.yahoo.jdisc.http.servlet.ServletRequest; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.Pkcs10CsrUtils; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; -import org.apache.http.client.ResponseHandler; -import org.apache.http.client.methods.HttpUriRequest; import org.junit.Before; import org.junit.Test; import javax.net.ssl.SSLContext; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.security.Principal; -import java.security.cert.X509Certificate; import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author mpolden @@ -38,8 +29,6 @@ public class CertificateAuthorityApiTest extends ContainerTester { private static final String INSTANCE_ID = "1.cluster1.default.app1.tenant1.us-north-1.prod.node"; private static final String INSTANCE_ID_WITH_SUFFIX = INSTANCE_ID + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; - private static final String INVALID_INSTANCE_ID = "1.cluster1.default.otherapp.othertenant.us-north-1.prod.node"; - private static final String INVALID_INSTANCE_ID_WITH_SUFFIX = INVALID_INSTANCE_ID + ".instanceid.athenz.dev-us-north-1.vespa.aws.oath.cloud"; @Before public void before() { @@ -55,47 +44,24 @@ public class CertificateAuthorityApiTest extends ContainerTester { Request.Method.POST)); // POST instance registration with ZTS client - var ztsClient = new TestZtsClient(new AthenzPrincipal(new AthenzService("vespa.external.tenant-host")), null, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); + var ztsClient = new DefaultZtsClient(URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); var instanceIdentity = ztsClient.registerInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), new AthenzService("vespa.external", "tenant"), - getAttestationData(), + "identity document generated by config server", csr); assertEquals("CN=Vespa CA", instanceIdentity.certificate().getIssuerX500Principal().getName()); } - private X509Certificate registerInstance() throws Exception { - // POST instance registration - var csr = CertificateTester.createCsr(List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/", - instanceRegistrationJson(csr), - Request.Method.POST)); - - // POST instance registration with ZTS client - var ztsClient = new TestZtsClient(new AthenzPrincipal(new AthenzService("vespa.external.tenant-host")), null, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - var instanceIdentity = ztsClient.registerInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService("vespa.external", "tenant"), - getAttestationData(), - csr); - return instanceIdentity.certificate(); - } - @Test public void refresh_instance() throws Exception { - // Register instance to get cert - var certificate = registerInstance(); - // POST instance refresh var csr = CertificateTester.createCsr(List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); - var principal = new AthenzPrincipal(new AthenzService("vespa.external.tenant")); - var request = new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/" + INSTANCE_ID, + assertIdentityResponse(new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/" + INSTANCE_ID, instanceRefreshJson(csr), - Request.Method.POST, - principal); - request.getAttributes().put(ServletRequest.JDISC_REQUEST_X509CERT, new X509Certificate[]{certificate}); - assertIdentityResponse(request); + Request.Method.POST)); // POST instance refresh with ZTS client - var ztsClient = new TestZtsClient(principal, certificate, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); + var ztsClient = new DefaultZtsClient(URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), new AthenzService("vespa.external", "tenant"), INSTANCE_ID, @@ -104,7 +70,7 @@ public class CertificateAuthorityApiTest extends ContainerTester { } @Test - public void invalid_requests() throws Exception { + public void invalid_requests() { // POST instance registration with missing fields assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/ failed: Missing required field 'provider'\"}", new Request("http://localhost:12345/ca/v1/instance/", @@ -125,28 +91,11 @@ public class CertificateAuthorityApiTest extends ContainerTester { Request.Method.POST)); // POST instance refresh where instanceId does not match CSR dnsName - var principal = new AthenzPrincipal(new AthenzService("vespa.external.tenant")); csr = CertificateTester.createCsr(List.of("node1.example.com", INSTANCE_ID_WITH_SUFFIX)); assertResponse(400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/foobar failed: Mismatch between instance ID in URL path and instance ID in CSR [instanceId=foobar,instanceIdFromCsr=1.cluster1.default.app1.tenant1.us-north-1.prod.node]\"}", new Request("http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/foobar", instanceRefreshJson(csr), - Request.Method.POST, - principal)); - - // POST instance refresh using zts client where client cert does not contain instanceid - var certificate = registerInstance(); - var ztsClient = new TestZtsClient(principal, certificate, URI.create("http://localhost:12345/ca/v1/"), SSLContext.getDefault()); - try { - var invalidCsr = CertificateTester.createCsr(List.of("node1.example.com", INVALID_INSTANCE_ID_WITH_SUFFIX)); - var instanceIdentity = ztsClient.refreshInstance(new AthenzService("vespa.external", "provider_prod_us-north-1"), - new AthenzService("vespa.external", "tenant"), - INSTANCE_ID, - invalidCsr); - fail("Refresh instance should have failed"); - } catch (Exception e) { - String expectedMessage = "Received error from ZTS: code=0, message=\"POST http://localhost:12345/ca/v1/instance/vespa.external.provider_prod_us-north-1/vespa.external/tenant/1.cluster1.default.app1.tenant1.us-north-1.prod.node failed: Mismatch between instance ID in URL path and instance ID in CSR [instanceId=1.cluster1.default.app1.tenant1.us-north-1.prod.node,instanceIdFromCsr=1.cluster1.default.otherapp.othertenant.us-north-1.prod.node]\""; - assertEquals(expectedMessage, e.getMessage()); - } + Request.Method.POST)); } private void setCaCertificateAndKey() { @@ -183,52 +132,10 @@ public class CertificateAuthorityApiTest extends ContainerTester { " \"provider\": \"vespa.external.provider_prod_us-north-1\",\n" + " \"domain\": \"vespa.external\",\n" + " \"service\": \"tenant\",\n" + - " \"attestationData\": \""+getAttestationData()+"\",\n" + + " \"attestationData\": \"identity document generated by config server\",\n" + " \"csr\": \"" + csrPem + "\"\n" + "}"; return json.getBytes(StandardCharsets.UTF_8); } - private static String getAttestationData () { - var json = "{\n" + - " \"signature\": \"SIGNATURE\",\n" + - " \"signing-key-version\": 0,\n" + - " \"provider-unique-id\": \"0.default.default.application.tenant.us-north-1.dev.tenant\",\n" + - " \"provider-service\": \"domain.service\",\n" + - " \"document-version\": 1,\n" + - " \"configserver-hostname\": \"localhost\",\n" + - " \"instance-hostname\": \"docker-container\",\n" + - " \"created-at\": 1572000079.00000,\n" + - " \"ip-addresses\": [\n" + - " \"::1\"\n" + - " ],\n" + - " \"identity-type\": \"tenant\"\n" + - "}"; - return StringUtilities.escape(json); - } - - /* - Zts client that adds principal as header (since setting up ssl in test is cumbersome) - */ - private static class TestZtsClient extends DefaultZtsClient { - - private final Principal principal; - private final X509Certificate certificate; - - public TestZtsClient(Principal principal, X509Certificate certificate, URI ztsUrl, SSLContext sslContext) { - super(ztsUrl, sslContext); - this.principal = principal; - this.certificate = certificate; - } - - @Override - protected <T> T execute(HttpUriRequest request, ResponseHandler<T> responseHandler) { - request.addHeader("PRINCIPAL", principal.getName()); - Optional.ofNullable(certificate).ifPresent(cert -> { - var pem = X509CertificateUtils.toPem(certificate); - request.addHeader("CERTIFICATE", StringUtilities.escape(pem)); - }); - return super.execute(request, responseHandler); - } - } } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java index 0eda6bd946b..139314b0f86 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java @@ -67,20 +67,13 @@ public class ContainerTester { " <ztsUrl>https://localhost:123/</ztsUrl>\n" + " </config>\n" + " <component id='com.yahoo.vespa.hosted.ca.restapi.mock.SecretStoreMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.ca.restapi.mock.InstanceValidatorMock'/>\n" + " <handler id='com.yahoo.vespa.hosted.ca.restapi.CertificateAuthorityApiHandler'>\n" + " <binding>http://*/ca/v1/*</binding>\n" + " </handler>\n" + " <http>\n" + " <server id='default' port='12345'/>\n" + - " <filtering>\n" + - " <request-chain id=\"my-default-chain\">\n" + - " <filter id='com.yahoo.vespa.hosted.ca.restapi.mock.PrincipalFromHeaderFilter' />\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " </filtering>\n" + " </http>\n" + "</container>"; } -}
\ No newline at end of file +} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java index 545d7d9eab7..83ea9249ad0 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java @@ -4,12 +4,6 @@ package com.yahoo.vespa.hosted.ca.restapi; import com.yahoo.security.Pkcs10CsrUtils; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.ca.CertificateTester; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; @@ -20,8 +14,6 @@ import org.junit.Test; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.Collections; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -35,27 +27,15 @@ public class InstanceSerializerTest { public void deserialize_instance_registration() { var csr = CertificateTester.createCsr(); var csrPem = Pkcs10CsrUtils.toPem(csr); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - "signature", - 0, - new VespaUniqueInstanceId(0, "cluster", "instance", "application", "tenant", "region", "prod", IdentityType.NODE), - new AthenzService("domain", "service"), - 0, - "configserverhostname", - "instancehostname", - Instant.ofEpochSecond(1572000079), - Collections.emptySet(), - IdentityType.NODE); - - var json = String.format("{\n" + + var json = "{\n" + " \"provider\": \"provider_prod_us-north-1\",\n" + " \"domain\": \"vespa.external\",\n" + " \"service\": \"tenant\",\n" + - " \"attestationData\":\"%s\",\n" + + " \"attestationData\": \"identity document from configserevr\",\n" + " \"csr\": \"" + csrPem + "\"\n" + - "}", StringUtilities.escape(EntityBindingsMapper.toAttestationData(signedIdentityDocument))); + "}"; var instanceRegistration = new InstanceRegistration("provider_prod_us-north-1", "vespa.external", - "tenant", signedIdentityDocument, + "tenant", "identity document from configserevr", csr); var deserialized = InstanceSerializer.registrationFromSlime(SlimeUtils.jsonToSlime(json)); assertEquals(instanceRegistration, deserialized); diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java deleted file mode 100644 index 9c1d4c49b07..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/InstanceValidatorMock.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator; - -/** - * @author mortent - */ -public class InstanceValidatorMock extends InstanceValidator { - - public InstanceValidatorMock() { - super(null, null, null, null, null); - } - - @Override - public boolean isValidInstance(InstanceConfirmation instanceConfirmation) { - return instanceConfirmation.attributes.get(SAN_DNS_ATTRNAME) != null && - instanceConfirmation.attributes.get(SAN_IPS_ATTRNAME) != null; - } - - @Override - public boolean isValidRefresh(InstanceConfirmation confirmation) { - return confirmation.attributes.get(SAN_DNS_ATTRNAME) != null && - confirmation.attributes.get(SAN_IPS_ATTRNAME) != null; - } -} diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java deleted file mode 100644 index d9ee4c8bb9b..00000000000 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/mock/PrincipalFromHeaderFilter.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.ca.restapi.mock; - -import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.jdisc.http.filter.DiscFilterRequest; -import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.jdisc.http.servlet.ServletRequest; -import com.yahoo.security.X509CertificateUtils; -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.athenz.api.AthenzPrincipal; -import com.yahoo.vespa.athenz.api.AthenzService; - -import java.security.cert.X509Certificate; -import java.util.Optional; - -/** - * Read principal from http header - * - * @author mortent - */ -public class PrincipalFromHeaderFilter implements SecurityRequestFilter { - - @Override - public void filter(DiscFilterRequest request, ResponseHandler handler) { - String principal = request.getHeader("PRINCIPAL"); - request.setUserPrincipal(new AthenzPrincipal(new AthenzService(principal))); - - Optional<String> certificate = Optional.ofNullable(request.getHeader("CERTIFICATE")); - certificate.ifPresent(cert -> { - var x509cert = X509CertificateUtils.fromPem(StringUtilities.unescape(cert)); - request.setAttribute(ServletRequest.JDISC_REQUEST_X509CERT, new X509Certificate[]{x509cert}); - }); - } -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java index 26fdddb01e5..74f3cc276a5 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java @@ -132,7 +132,7 @@ public class ClusterStateGenerator { final Map<Node, NodeStateReason> nodeStateReasons = new HashMap<>(); for (final NodeInfo nodeInfo : cluster.getNodeInfo()) { - final NodeState nodeState = computeEffectiveNodeState(nodeInfo, params); + final NodeState nodeState = computeEffectiveNodeState(nodeInfo, params, nodeStateReasons); workingState.setNodeState(nodeInfo.getNode(), nodeState); } @@ -159,7 +159,10 @@ public class ClusterStateGenerator { baseline.setDescription(wanted.getDescription()); } - private static NodeState computeEffectiveNodeState(final NodeInfo nodeInfo, final Params params) { + private static NodeState computeEffectiveNodeState(final NodeInfo nodeInfo, + final Params params, + Map<Node, NodeStateReason> nodeStateReasons) + { final NodeState reported = nodeInfo.getReportedState(); final NodeState wanted = nodeInfo.getWantedState(); final NodeState baseline = reported.clone(); @@ -171,7 +174,7 @@ public class ClusterStateGenerator { baseline.setStartTimestamp(0); } if (nodeInfo.isStorage()) { - applyStorageSpecificStateTransforms(nodeInfo, params, reported, wanted, baseline); + applyStorageSpecificStateTransforms(nodeInfo, params, reported, wanted, baseline, nodeStateReasons); } if (baseline.above(wanted)) { applyWantedStateToBaselineState(baseline, wanted); @@ -181,7 +184,8 @@ public class ClusterStateGenerator { } private static void applyStorageSpecificStateTransforms(NodeInfo nodeInfo, Params params, NodeState reported, - NodeState wanted, NodeState baseline) + NodeState wanted, NodeState baseline, + Map<Node, NodeStateReason> nodeStateReasons) { if (reported.getState() == State.INITIALIZING) { if (timedOutWithoutNewInitProgress(reported, nodeInfo, params) @@ -195,7 +199,7 @@ public class ClusterStateGenerator { } } // TODO ensure that maintenance cannot override Down for any other cases - if (withinTemporalMaintenancePeriod(nodeInfo, baseline, params) && wanted.getState() != State.DOWN) { + if (withinTemporalMaintenancePeriod(nodeInfo, baseline, nodeStateReasons, params) && wanted.getState() != State.DOWN) { baseline.setState(State.MAINTENANCE); } } @@ -244,13 +248,18 @@ public class ClusterStateGenerator { */ private static boolean withinTemporalMaintenancePeriod(final NodeInfo nodeInfo, final NodeState baseline, + Map<Node, NodeStateReason> nodeStateReasons, final Params params) { final Integer transitionTime = params.transitionTimes.get(nodeInfo.getNode().getType()); if (transitionTime == 0 || !baseline.getState().oneOf("sd")) { return false; } - return nodeInfo.getTransitionTime() + transitionTime > params.currentTimeInMillis; + if (nodeInfo.getTransitionTime() + transitionTime > params.currentTimeInMillis) { + return true; + } + nodeStateReasons.put(nodeInfo.getNode(), NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD); + return false; } private static void takeDownGroupsWithTooLowAvailability(final ClusterState workingState, diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java index cadc065dd51..2f065a9ba75 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java @@ -31,6 +31,7 @@ public class EventDiffCalculator { ClusterStateBundle fromState; ClusterStateBundle toState; long currentTime; + long maxMaintenanceGracePeriodTimeMs; public Params cluster(ContentCluster cluster) { this.cluster = cluster; @@ -48,6 +49,10 @@ public class EventDiffCalculator { this.currentTime = time; return this; } + public Params maxMaintenanceGracePeriodTimeMs(long timeMs) { + this.maxMaintenanceGracePeriodTimeMs = timeMs; + return this; + } } public static Params params() { return new Params(); } @@ -58,17 +63,20 @@ public class EventDiffCalculator { final AnnotatedClusterState fromState; final AnnotatedClusterState toState; final long currentTime; + final long maxMaintenanceGracePeriodTimeMs; PerStateParams(ContentCluster cluster, Optional<String> bucketSpace, AnnotatedClusterState fromState, AnnotatedClusterState toState, - long currentTime) { + long currentTime, + long maxMaintenanceGracePeriodTimeMs) { this.cluster = cluster; this.bucketSpace = bucketSpace; this.fromState = fromState; this.toState = toState; this.currentTime = currentTime; + this.maxMaintenanceGracePeriodTimeMs = maxMaintenanceGracePeriodTimeMs; } } @@ -86,7 +94,8 @@ public class EventDiffCalculator { Optional.empty(), params.fromState.getBaselineAnnotatedState(), params.toState.getBaselineAnnotatedState(), - params.currentTime); + params.currentTime, + params.maxMaintenanceGracePeriodTimeMs); } private static void emitWholeClusterDiffEvent(final PerStateParams params, final List<Event> events) { @@ -137,11 +146,10 @@ public class EventDiffCalculator { final NodeState nodeTo = toState.getNodeState(n); if (!nodeTo.equals(nodeFrom)) { final NodeInfo info = cluster.getNodeInfo(n); - events.add(createNodeEvent(info, String.format("Altered node state in cluster state from '%s' to '%s'", - nodeFrom.toString(true), nodeTo.toString(true)), params)); - NodeStateReason prevReason = params.fromState.getNodeStateReasons().get(n); NodeStateReason currReason = params.toState.getNodeStateReasons().get(n); + // Add specific reason events for node edge _before_ the actual transition event itself. + // This makes the timeline of events more obvious. if (isGroupDownEdge(prevReason, currReason)) { events.add(createNodeEvent(info, "Group node availability is below configured threshold", params)); } else if (isGroupUpEdge(prevReason, currReason)) { @@ -150,7 +158,12 @@ public class EventDiffCalculator { events.add(createNodeEvent(info, "Node may have merges pending", params)); } else if (isMayHaveMergesPendingDownEdge(prevReason, currReason)) { events.add(createNodeEvent(info, "Node no longer has merges pending", params)); + } else if (isMaintenanceGracePeriodExceededDownEdge(prevReason, currReason, nodeFrom, nodeTo)) { + events.add(createNodeEvent(info, String.format("Exceeded implicit maintenance mode grace period of " + + "%d milliseconds. Marking node down.", params.maxMaintenanceGracePeriodTimeMs), params)); } + events.add(createNodeEvent(info, String.format("Altered node state in cluster state from '%s' to '%s'", + nodeFrom.toString(true), nodeTo.toString(true)), params)); } } @@ -178,6 +191,14 @@ public class EventDiffCalculator { return prevReason == NodeStateReason.MAY_HAVE_MERGES_PENDING && currReason != NodeStateReason.MAY_HAVE_MERGES_PENDING; } + private static boolean isMaintenanceGracePeriodExceededDownEdge(NodeStateReason prevReason, NodeStateReason currReason, + NodeState prevState, NodeState currState) { + return (prevReason != NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD && + currReason == NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD && + prevState.getState() == State.MAINTENANCE && + currState.getState() == State.DOWN); + } + private static boolean clusterHasTransitionedToUpState(ClusterState prevState, ClusterState currentState) { return prevState.getClusterState() != State.UP && currentState.getClusterState() == State.UP; } @@ -207,7 +228,8 @@ public class EventDiffCalculator { Optional.of(bucketSpace), fromDerivedState, toDerivedState, - params.currentTime); + params.currentTime, + params.maxMaintenanceGracePeriodTimeMs); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 364184331a8..cbff11af730 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -812,8 +812,9 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd // Send getNodeState requests to zero or more nodes. didWork |= stateGatherer.sendMessages(cluster, communicator, this); - // Important: timer events must use a consolidated state, or they might trigger edge events multiple times. - didWork |= stateChangeHandler.watchTimers(cluster, consolidatedClusterState(), this); + // Important: timer events must use a state with pending changes visible, or they might + // trigger edge events multiple times. + didWork |= stateChangeHandler.watchTimers(cluster, stateVersionTracker.getLatestCandidateState().getClusterState(), this); didWork |= recomputeClusterStateIfRequired(); @@ -928,7 +929,8 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd .cluster(cluster) .fromState(fromState) .toState(toState) - .currentTimeMs(timeNowMs)); + .currentTimeMs(timeNowMs) + .maxMaintenanceGracePeriodTimeMs(options.storageNodeMaxTransitionTimeMs())); for (Event event : deltaEvents) { eventLog.add(event, isMaster); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index f49b626d347..5e9e91e1cb6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -157,6 +157,10 @@ public class FleetControllerOptions implements Cloneable { this.maxDeferredTaskVersionWaitTime = maxDeferredTaskVersionWaitTime; } + public long storageNodeMaxTransitionTimeMs() { + return maxTransitionTime.getOrDefault(NodeType.STORAGE, 10_000); + } + public FleetControllerOptions clone() { try { // TODO: This should deep clone diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java index 7a6be664ec8..3f550724cef 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateReason.java @@ -6,8 +6,8 @@ public enum NodeStateReason { // FIXME some of these reasons may be unnecessary as they are reported implicitly by reported/wanted state changes NODE_TOO_UNSTABLE, WITHIN_MAINTENANCE_GRACE_PERIOD, + NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD, FORCED_INTO_MAINTENANCE, GROUP_IS_DOWN, MAY_HAVE_MERGES_PENDING - } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 3f38ea6c018..3c19f70d1e2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -305,10 +305,6 @@ public class StateChangeHandler { if (nodeStillUnavailableAfterTransitionTimeExceeded( currentTime, node, currentStateInSystem, lastReportedState)) { - eventLog.add(NodeEvent.forBaseline(node, String.format( - "%d milliseconds without contact. Marking node down.", - currentTime - node.getTransitionTime()), - NodeEvent.Type.CURRENT, currentTime), isMaster); triggeredAnyTimers = true; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java index 2f91e569b89..08329c874b5 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java @@ -12,7 +12,9 @@ import java.util.Optional; import static com.yahoo.vespa.clustercontroller.core.matchers.HasStateReasonForNode.hasStateReasonForNode; import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.storageNode; +import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.distributorNode; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; @@ -340,6 +342,8 @@ public class ClusterStateGeneratorTest { final AnnotatedClusterState state = ClusterStateGenerator.generatedStateFrom(params); assertThat(state.toString(), equalTo("distributor:5 storage:5 .1.s:d")); + assertThat(state.getNodeStateReasons(), + hasStateReasonForNode(storageNode(1), NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD)); } @Test @@ -355,6 +359,8 @@ public class ClusterStateGeneratorTest { final AnnotatedClusterState state = ClusterStateGenerator.generatedStateFrom(params); assertThat(state.toString(), equalTo("distributor:5 .2.s:d storage:5")); + assertThat(state.getNodeStateReasons(), + not(hasStateReasonForNode(distributorNode(1), NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD))); } @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java index e7c4bbfcaa8..ab8d73be99d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java @@ -32,6 +32,7 @@ public class EventDiffCalculatorTest { Map<String, AnnotatedClusterState.Builder> derivedBefore = new HashMap<>(); Map<String, AnnotatedClusterState.Builder> derivedAfter = new HashMap<>(); long currentTimeMs = 0; + long maxMaintenanceGracePeriodTimeMs = 10_000; EventFixture(int nodeCount) { this.clusterFixture = ClusterFixture.forFlatCluster(nodeCount); @@ -65,6 +66,10 @@ public class EventDiffCalculatorTest { this.currentTimeMs = timeMs; return this; } + EventFixture maxMaintenanceGracePeriodTimeMs(long timeMs) { + this.maxMaintenanceGracePeriodTimeMs = timeMs; + return this; + } EventFixture derivedClusterStateBefore(String bucketSpace, String stateStr) { getBuilder(derivedBefore, bucketSpace).clusterState(stateStr); return this; @@ -91,7 +96,8 @@ public class EventDiffCalculatorTest { .cluster(clusterFixture.cluster()) .fromState(ClusterStateBundle.of(baselineBefore.build(), toDerivedStates(derivedBefore))) .toState(ClusterStateBundle.of(baselineAfter.build(), toDerivedStates(derivedAfter))) - .currentTimeMs(currentTimeMs)); + .currentTimeMs(currentTimeMs) + .maxMaintenanceGracePeriodTimeMs(maxMaintenanceGracePeriodTimeMs)); } private static Map<String, AnnotatedClusterState> toDerivedStates(Map<String, AnnotatedClusterState.Builder> derivedBuilders) { @@ -407,4 +413,35 @@ public class EventDiffCalculatorTest { nodeEventWithDescription("Altered node state in cluster state from 'U' to 'M'")))); } + @Test + public void storage_node_passed_maintenance_grace_period_emits_event() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3 .0.s:m") + .clusterStateAfter("distributor:3 storage:3 .0.s:d") + .maxMaintenanceGracePeriodTimeMs(123_456) + .storageNodeReasonAfter(0, NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD); + + final List<Event> events = fixture.computeEventDiff(); + // Down edge event + event explaining why the node went down + assertThat(events.size(), equalTo(2)); + assertThat(events, hasItem(allOf( + eventForNode(storageNode(0)), + nodeEventWithDescription("Exceeded implicit maintenance mode grace period of 123456 milliseconds. Marking node down."), + nodeEventForBaseline()))); + } + + @Test + public void storage_node_maintenance_grace_period_event_only_emitted_on_maintenance_to_down_edge() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3 .0.s:u") + .clusterStateAfter("distributor:3 storage:3 .0.s:d") + .maxMaintenanceGracePeriodTimeMs(123_456) + .storageNodeReasonAfter(0, NodeStateReason.NODE_NOT_BACK_UP_WITHIN_GRACE_PERIOD); + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(allOf( + eventForNode(storageNode(0)), + nodeEventForBaseline()))); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 016cce9580e..2c8220c0dba 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -247,7 +247,7 @@ public class StateChangeTest extends FleetControllerTest { "Event: storage.0: Failed to get node state: D: Closed at other end\n" + "Event: storage.0: Stopped or possibly crashed after 1000 ms, which is before stable state time period. Premature crash count is now 1.\n" + "Event: storage.0: Altered node state in cluster state from 'U' to 'M: Closed at other end'\n" + - "Event: storage.0: 5001 milliseconds without contact. Marking node down.\n" + + "Event: storage.0: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + "Event: storage.0: Altered node state in cluster state from 'M: Closed at other end' to 'D: Closed at other end'\n" + "Event: storage.0: Now reporting state U, t 12345679\n" + "Event: storage.0: Altered node state in cluster state from 'D: Closed at other end' to 'U, t 12345679'\n"); @@ -326,7 +326,7 @@ public class StateChangeTest extends FleetControllerTest { "Event: storage.0: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + "Event: storage.0: Failed to get node state: D: controlled shutdown\n" + "Event: storage.0: Altered node state in cluster state from 'U' to 'M: controlled shutdown'\n" + - "Event: storage.0: 5001 milliseconds without contact. Marking node down.\n" + + "Event: storage.0: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + "Event: storage.0: Altered node state in cluster state from 'M: controlled shutdown' to 'D: controlled shutdown'\n" + "Event: storage.0: Now reporting state U\n" + "Event: storage.0: Altered node state in cluster state from 'D: controlled shutdown' to 'U'\n"); @@ -569,7 +569,7 @@ public class StateChangeTest extends FleetControllerTest { "Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + "Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end'\n" + - "Event: storage.6: 100000 milliseconds without contact. Marking node down.\n" + + "Event: storage.6: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + "Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'D: Connection error: Closed at other end'\n" + "Event: storage.6: Now reporting state I, i 0.00100 (ls)\n" + "Event: storage.6: Now reporting state I, i 0.100 (read)\n" + @@ -650,7 +650,7 @@ public class StateChangeTest extends FleetControllerTest { "Event: storage.6: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + "Event: storage.6: Failed to get node state: D: Connection error: Closed at other end\n" + "Event: storage.6: Altered node state in cluster state from 'U' to 'M: Connection error: Closed at other end'\n" + - "Event: storage.6: 1000000 milliseconds without contact. Marking node down.\n" + + "Event: storage.6: Exceeded implicit maintenance mode grace period of 5000 milliseconds. Marking node down.\n" + "Event: storage.6: Altered node state in cluster state from 'M: Connection error: Closed at other end' to 'D: Connection error: Closed at other end'\n" + "Event: storage.6: Now reporting state I, i 0.100 (read)\n" + "Event: storage.6: Altered node state in cluster state from 'D: Connection error: Closed at other end' to 'I, i 0.100 (read)'\n" + @@ -1213,8 +1213,11 @@ public class StateChangeTest extends FleetControllerTest { "Event: storage.2: Altered node state in cluster state from 'D: Node not seen in slobrok.' to 'U'\n" + "Event: storage.2: Failed to get node state: D: foo\n" + "Event: storage.2: Stopped or possibly crashed after 500 ms, which is before stable state time period. Premature crash count is now 1.\n" + - "Event: storage.2: Altered node state in cluster state from 'U' to 'M: foo'\n" + - "Event: storage.2: 5000 milliseconds without contact. Marking node down.\n"); + "Event: storage.2: Altered node state in cluster state from 'U' to 'M: foo'\n"); + // Note: even though max transition time has passed, events are now emitted only on cluster state + // publish edges. These are currently suppressed when the cluster state is down, as all cluster down + // states are considered similar to other cluster down states. This is not necessarily optimal, but + // if the cluster is down there are bigger problems than not having some debug events logged. } @Test diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java index cbbcee0dcfa..16a0454cb4b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.ImmutableSDField; -import java.util.Iterator; import java.util.LinkedHashSet; import java.util.Set; @@ -41,7 +40,7 @@ public class DefaultRankProfile extends RankProfile { RankSetting setting = super.getRankSetting(fieldOrIndex,type); if (setting != null) return setting; - SDField field = getSearch().getConcreteField(fieldOrIndex); + ImmutableSDField field = getSearch().getConcreteField(fieldOrIndex); if (field != null) { setting = toRankSetting(field,type); if (setting != null) @@ -58,7 +57,7 @@ public class DefaultRankProfile extends RankProfile { return null; } - private RankSetting toRankSetting(SDField field,RankSetting.Type type) { + private RankSetting toRankSetting(ImmutableSDField field,RankSetting.Type type) { if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight()>0 && field.getWeight()!=100) return new RankSetting(field.getName(),type,field.getWeight()); if (type.equals(RankSetting.Type.RANKTYPE)) @@ -90,7 +89,7 @@ public class DefaultRankProfile extends RankProfile { public Set<RankSetting> rankSettings() { Set<RankSetting> settings = new LinkedHashSet<>(20); settings.addAll(this.rankSettings); - for (SDField field : getSearch().allConcreteFields() ) { + for (ImmutableSDField field : getSearch().allConcreteFields() ) { addSetting(field, RankSetting.Type.WEIGHT, settings); addSetting(field, RankSetting.Type.RANKTYPE, settings); addSetting(field, RankSetting.Type.LITERALBOOST, settings); @@ -104,7 +103,7 @@ public class DefaultRankProfile extends RankProfile { return settings; } - private void addSetting(SDField field, RankSetting.Type type, Set<RankSetting> settings) { + private void addSetting(ImmutableSDField field, RankSetting.Type type, Set<RankSetting> settings) { if (type.isIndexLevel()) { addIndexSettings(field, type, settings); } @@ -115,14 +114,12 @@ public class DefaultRankProfile extends RankProfile { } } - private void addIndexSettings(SDField field, RankSetting.Type type, Set<RankSetting> settings) { - for (Iterator i = field.getFieldNameAsIterator(); i.hasNext(); ) { - String indexName = (String)i.next(); + private void addIndexSettings(ImmutableSDField field, RankSetting.Type type, Set<RankSetting> settings) { + String indexName = field.getName(); - // TODO: Make a ranking object in the index override the field level ranking object - if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { - settings.add(new RankSetting(indexName, type, true)); - } + // TODO: Make a ranking object in the index override the field level ranking object + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { + settings.add(new RankSetting(indexName, type, true)); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 1e133d0b8f4..0d10932c333 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -20,7 +20,7 @@ public class FeatureNames { } public static Reference asAttributeFeature(String attributeName) { - return Reference.simple("attribute", quoteIfNecessary(attributeName)); + return Reference.simple("attribute", attributeName); } public static Reference asQueryFeature(String propertyName) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java index abe19ddf831..5e580d8f4df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java @@ -10,8 +10,8 @@ import com.yahoo.searchdefinition.document.FieldSet; /** * The field sets owned by a {@link Search} * Both built in and user defined. - * @author vegardh * + * @author vegardh */ public class FieldSets { @@ -46,18 +46,12 @@ public class FieldSets { builtInFieldSets.get(setName).addFieldName(field); } - /** - * The built in field sets, unmodifiable - * @return built in field sets - */ + /** Returns the built in field sets, unmodifiable */ public Map<String, FieldSet> builtInFieldSets() { return Collections.unmodifiableMap(builtInFieldSets); } - /** - * The user defined field sets, unmodifiable - * @return user field sets - */ + /** Returns the user defined field sets, unmodifiable */ public Map<String, FieldSet> userFieldSets() { return Collections.unmodifiableMap(userFieldSets); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java index 795ec9badbb..0b9447d05f5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java @@ -1,9 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; +import java.io.Reader; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -15,11 +19,23 @@ import java.util.stream.Stream; */ public interface ImmutableSearch { + String getName(); + Index getIndex(String name); + ImmutableSDField getConcreteField(String name); + //TODO split in mutating/immutable by returning List<ImmutableSDField> + List<SDField> allConcreteFields(); + List<Index> getExplicitIndices(); + Reader getRankingExpression(String fileName); + ApplicationPackage applicationPackage(); + RankingConstants rankingConstants(); Stream<ImmutableSDField> allImportedFields(); ImmutableSDField getField(String name); - Stream<ImmutableSDField> allFields(); + default Stream<ImmutableSDField> allFields() { + return allFieldsList().stream(); + } + List<ImmutableSDField> allFieldsList(); Map<String, SummaryField> getSummaryFields(ImmutableSDField field); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index 0ea3f5c24a3..e46db1d1b5f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -16,14 +16,15 @@ import java.util.Set; * Two indices are equal if they have the same name and the same settings, except * alias settings (which are excluded). * - * @author bratseth + * @author bratseth */ public class Index implements Cloneable, Serializable { - public static enum Type { + public enum Type { + VESPA("vespa"); private String name; - private Type(String name) { this.name = name; } + Type(String name) { this.name = name; } public String getName() { return name; } } @@ -34,7 +35,7 @@ public class Index implements Cloneable, Serializable { private String name; /** The rank type of this index */ - private RankType rankType=null; + private RankType rankType = null; /** Whether this index supports prefix search */ private boolean prefix; @@ -46,10 +47,10 @@ public class Index implements Cloneable, Serializable { * The stemming setting of this field, or null to use the default. * Default is determined by the owning search definition. */ - private Stemming stemming=null; + private Stemming stemming = null; /** Whether the content of this index is normalized */ - private boolean normalized=true; + private boolean normalized = true; private Type type = Type.VESPA; @@ -64,16 +65,16 @@ public class Index implements Cloneable, Serializable { } public Index(String name, boolean prefix) { - this.name=name; - this.prefix=prefix; + this.name = name; + this.prefix = prefix; } - public void setName(String name) { this.name=name; } + public void setName(String name) { this.name = name; } public String getName() { return name; } /** Sets the rank type of this field */ - public void setRankType(RankType rankType) { this.rankType=rankType; } + public void setRankType(RankType rankType) { this.rankType = rankType; } /** Returns the rank type of this field, or null if nothing is set */ public RankType getRankType() { return rankType; } @@ -86,7 +87,7 @@ public class Index implements Cloneable, Serializable { * this is never null */ public Stemming getStemming(Search search) { - if (stemming!=null) + if (stemming != null) return stemming; else return search.getStemming(); @@ -95,7 +96,7 @@ public class Index implements Cloneable, Serializable { /** * Sets how this field should be stemmed, or set to null to use the default. */ - public void setStemming(Stemming stemming) { this.stemming=stemming; } + public void setStemming(Stemming stemming) { this.stemming = stemming; } /** Returns whether this index supports prefix search, default is false */ public boolean isPrefix() { return prefix; } @@ -113,10 +114,12 @@ public class Index implements Cloneable, Serializable { return Collections.unmodifiableSet(aliases).iterator(); } + @Override public int hashCode() { return name.hashCode() + ( prefix ? 17 : 0 ); } + @Override public boolean equals(Object object) { if ( ! (object instanceof Index)) return false; @@ -137,6 +140,7 @@ public class Index implements Cloneable, Serializable { } /** Makes a deep copy of this index */ + @Override public Object clone() { try { Index copy=(Index)super.clone(); @@ -152,34 +156,22 @@ public class Index implements Cloneable, Serializable { return (Index)clone(); } - /** - * The index engine type - * @return the type - */ + /** Returns the index engine type */ public Type getType() { return type; } - /** - * Sets the index engine type - * @param type a index engine type - */ + /** Sets the index engine type */ public void setType(Type type) { this.type = type; } - /** - * The boolean index definition - * @return the boolean index definition - */ + /** Returns the boolean index definition */ public BooleanIndexDefinition getBooleanIndexDefiniton() { return boolIndex; } - /** - * Sets the boolean index definition - * @param def boolean index definition - */ + /** Sets the boolean index definition */ public void setBooleanIndexDefiniton(BooleanIndexDefinition def) { boolIndex = def; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index 6109e5c4aae..a54e21aae68 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -42,8 +42,9 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement /** For invocation loop detection */ private final Deque<Reference> currentResolutionCallStack; - MapEvaluationTypeContext(Collection<ExpressionFunction> functions) { + MapEvaluationTypeContext(Collection<ExpressionFunction> functions, Map<Reference, TensorType> featureTypes) { super(functions); + this.featureTypes.putAll(featureTypes); this.currentResolutionCallStack = new ArrayDeque<>(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 5c43428a5d7..1283da20395 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -49,13 +49,13 @@ import java.util.stream.Stream; * * @author bratseth */ -public class RankProfile implements Serializable, Cloneable { +public class RankProfile implements Cloneable { /** The search definition-unique name of this rank profile */ private final String name; /** The search definition owning this profile, or null if global (owned by a model) */ - private final Search search; + private final ImmutableSearch search; /** The model owning this profile if it is global, or null if it is owned by a search definition */ private final VespaModel model; @@ -64,7 +64,7 @@ public class RankProfile implements Serializable, Cloneable { private String inheritedName = null; /** The match settings of this profile */ - protected MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhaseSettings = null; /** The rank settings of this profile */ protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); @@ -112,6 +112,8 @@ public class RankProfile implements Serializable, Cloneable { private final TypeSettings queryFeatureTypes = new TypeSettings(); + private List<ImmutableSDField> allFieldsList; + /** * Creates a new rank profile for a particular search definition * @@ -143,7 +145,7 @@ public class RankProfile implements Serializable, Cloneable { public String getName() { return name; } /** Returns the search definition owning this, or null if it is global */ - public Search getSearch() { return search; } + public ImmutableSearch getSearch() { return search; } /** Returns the application this is part of */ public ApplicationPackage applicationPackage() { @@ -156,7 +158,11 @@ public class RankProfile implements Serializable, Cloneable { } private Stream<ImmutableSDField> allFields() { - return search != null ? search.allFields() : Stream.empty(); + if (search == null) return Stream.empty(); + if (allFieldsList == null) { + allFieldsList = search.allFieldsList(); + } + return allFieldsList.stream(); } private Stream<ImmutableSDField> allImportedFields() { @@ -237,7 +243,7 @@ public class RankProfile implements Serializable, Cloneable { * @param type the type that the field is required to be. * @return the rank setting found, or null. */ - public RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { + RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) { RankSetting setting = i.next(); if (setting.getFieldName().equals(field) && @@ -342,7 +348,7 @@ public class RankProfile implements Serializable, Cloneable { return null; } - public void setFirstPhaseRanking(RankingExpression rankingExpression) { + void setFirstPhaseRanking(RankingExpression rankingExpression) { this.firstPhaseRanking = rankingExpression; } @@ -385,7 +391,7 @@ public class RankProfile implements Serializable, Cloneable { return Collections.emptySet(); } - public void addSummaryFeature(ReferenceNode feature) { + private void addSummaryFeature(ReferenceNode feature) { if (summaryFeatures == null) summaryFeatures = new LinkedHashSet<>(); summaryFeatures.add(feature); @@ -409,7 +415,7 @@ public class RankProfile implements Serializable, Cloneable { return Collections.emptySet(); } - public void addRankFeature(ReferenceNode feature) { + private void addRankFeature(ReferenceNode feature) { if (rankFeatures == null) rankFeatures = new LinkedHashSet<>(); rankFeatures.add(feature); @@ -669,15 +675,16 @@ public class RankProfile implements Serializable, Cloneable { checkNameCollisions(getFunctions(), getConstants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); + Map<Reference, TensorType> featureTypes = collectFeatureTypes(); // Function compiling first pass: compile inline functions without resolving other functions Map<String, RankingExpressionFunction> inlineFunctions = - compileFunctions(this::getInlineFunctions, queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); + compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions - functions = compileFunctions(this::getFunctions, queryProfiles, importedModels, inlineFunctions, expressionTransforms); + functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { @@ -695,6 +702,7 @@ public class RankProfile implements Serializable, Cloneable { private Map<String, RankingExpressionFunction> compileFunctions(Supplier<Map<String, RankingExpressionFunction>> functions, QueryProfileRegistry queryProfiles, + Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { @@ -705,7 +713,7 @@ public class RankProfile implements Serializable, Cloneable { // A straightforward iteration will either miss those functions, or may cause a ConcurrentModificationException while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); - RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, + RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled)); } @@ -723,6 +731,7 @@ public class RankProfile implements Serializable, Cloneable { private RankingExpression compile(RankingExpression expression, QueryProfileRegistry queryProfiles, + Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, Map<String, Value> constants, Map<String, RankingExpressionFunction> inlineFunctions, @@ -730,6 +739,7 @@ public class RankProfile implements Serializable, Cloneable { if (expression == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, + featureTypes, importedModels, constants, inlineFunctions); @@ -745,18 +755,28 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles) { + + return typeContext(queryProfiles, collectFeatureTypes()); + } + + private Map<Reference, TensorType> collectFeatureTypes() { + Map<Reference, TensorType> featureTypes = new HashMap<>(); + // Add attributes + allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); + allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); + return featureTypes; + } + + public MapEvaluationTypeContext typeContext(QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getFunctions().values().stream() .map(RankingExpressionFunction::function) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + featureTypes); // Add small and large constants, respectively getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); rankingConstants().asMap().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType())); - // Add attributes - allFields().forEach(field -> addAttributeFeatureTypes(field, context)); - allImportedFields().forEach(field -> addAttributeFeatureTypes(field, context)); - // Add query features from rank profile types reached from the "default" profile for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { for (FieldDescription field : queryProfileType.declaredFields().values()) { @@ -779,13 +799,13 @@ public class RankProfile implements Serializable, Cloneable { return context; } - private void addAttributeFeatureTypes(ImmutableSDField field, MapEvaluationTypeContext context) { + private void addAttributeFeatureTypes(ImmutableSDField field, Map<Reference, TensorType> featureTypes) { Attribute attribute = field.getAttribute(); field.getAttributes().forEach((k, a) -> { String name = k; if (attribute == a) // this attribute should take the fields name name = field.getName(); // switch to that - it is separate for imported fields - context.setType(FeatureNames.asAttributeFeature(name), + featureTypes.put(FeatureNames.asAttributeFeature(name), a.tensorType().orElse(TensorType.empty)); }); } @@ -925,7 +945,7 @@ public class RankProfile implements Serializable, Cloneable { /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ private final boolean inline; - public RankingExpressionFunction(ExpressionFunction function, boolean inline) { + RankingExpressionFunction(ExpressionFunction function, boolean inline) { this.function = function; this.inline = inline; } @@ -940,7 +960,7 @@ public class RankProfile implements Serializable, Cloneable { return inline && function.arguments().isEmpty(); // only inline no-arg functions; } - public RankingExpressionFunction withExpression(RankingExpression expression) { + RankingExpressionFunction withExpression(RankingExpression expression) { return new RankingExpressionFunction(function.withBody(expression), inline); } @@ -968,7 +988,7 @@ public class RankProfile implements Serializable, Cloneable { public double getCutoffFactor() { return cutoffFactor; } public Diversity.CutoffStrategy getCutoffStrategy() { return cutoffStrategy; } - public void checkValid() { + void checkValid() { if (attribute == null || attribute.isEmpty()) { throw new IllegalArgumentException("'diversity' did not set non-empty diversity attribute name."); } @@ -1026,7 +1046,7 @@ public class RankProfile implements Serializable, Cloneable { private final Map<String, String> types = new HashMap<>(); - public void addType(String name, String type) { + void addType(String name, String type) { types.put(name, type); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 53afebfd93b..bf585df9005 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -20,8 +20,8 @@ import java.util.Set; */ public class RankProfileRegistry { - private final Map<RankProfile, Search> rankProfileToSearch = new LinkedHashMap<>(); - private final Map<Search, Map<String, RankProfile>> rankProfiles = new LinkedHashMap<>(); + private final Map<RankProfile, ImmutableSearch> rankProfileToSearch = new LinkedHashMap<>(); + private final Map<ImmutableSearch, Map<String, RankProfile>> rankProfiles = new LinkedHashMap<>(); /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); @@ -65,7 +65,7 @@ public class RankProfileRegistry { * @param name the name of the rank profile * @return the RankProfile to return. */ - public RankProfile get(Search search, String name) { + public RankProfile get(ImmutableSearch search, String name) { Map<String, RankProfile> profiles = rankProfiles.get(search); if (profiles == null) return null; return profiles.get(name); @@ -85,7 +85,7 @@ public class RankProfileRegistry { * @param search {@link Search} to get rank profiles for * @return a collection of {@link RankProfile} instances */ - public Collection<RankProfile> rankProfilesOf(Search search) { + public Collection<RankProfile> rankProfilesOf(ImmutableSearch search) { Map<String, RankProfile> mapping = rankProfiles.get(search); if (mapping == null) { return Collections.emptyList(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 81c549a6f78..f90a7e4f6cd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -6,6 +6,7 @@ import com.yahoo.document.Field; import com.yahoo.searchdefinition.derived.SummaryClass; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.document.ImportedFields; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; @@ -16,7 +17,6 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import java.io.Reader; -import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,7 +40,7 @@ import java.util.stream.Stream; // TODO: Make a class owned by this, for each of these responsibilities: // Managing indexes, managing attributes, managing summary classes. // Ensure that after the processing step, all implicit instances of the above types are explicitly represented -public class Search implements Serializable, ImmutableSearch { +public class Search implements ImmutableSearch { private static final Logger log = Logger.getLogger(Search.class.getName()); private static final String SD_DOC_FIELD_NAME = "sddocname"; @@ -53,7 +53,7 @@ public class Search implements Serializable, ImmutableSearch { return RESERVED_NAMES.contains(name); } - private FieldSets fieldSets = new FieldSets(); + private final FieldSets fieldSets = new FieldSets(); /** The unique name of this search definition */ private String name; @@ -68,26 +68,27 @@ public class Search implements Serializable, ImmutableSearch { private SDDocumentType docType; /** The extra fields of this search definition */ - private Map<String, SDField> fields = new LinkedHashMap<>(); + private final Map<String, SDField> fields = new LinkedHashMap<>(); /** The explicitly defined indices of this search definition */ - private Map<String, Index> indices = new LinkedHashMap<>(); + private final Map<String, Index> indices = new LinkedHashMap<>(); /** The explicitly defined summaries of this search definition. _Must_ preserve order. */ - private Map<String, DocumentSummary> summaries = new LinkedHashMap<>(); + private final Map<String, DocumentSummary> summaries = new LinkedHashMap<>(); /** Ranking constants of this */ - private RankingConstants rankingConstants = new RankingConstants(); + private final RankingConstants rankingConstants = new RankingConstants(); private Optional<TemporaryImportedFields> temporaryImportedFields = Optional.of(new TemporaryImportedFields()); private Optional<ImportedFields> importedFields = Optional.empty(); - private ApplicationPackage applicationPackage; + private final ApplicationPackage applicationPackage; /** * Creates a search definition which just holds a set of documents which should not (here, directly) be searchable */ protected Search() { + applicationPackage = null; documentsOnly = true; } @@ -106,6 +107,7 @@ public class Search implements Serializable, ImmutableSearch { this.name = name; } + @Override public String getName() { return name; } @@ -154,6 +156,7 @@ public class Search implements Serializable, ImmutableSearch { docType = document; } + @Override public RankingConstants rankingConstants() { return rankingConstants; } public Optional<TemporaryImportedFields> temporaryImportedFields() { @@ -188,12 +191,18 @@ public class Search implements Serializable, ImmutableSearch { } @Override - public Stream<ImmutableSDField> allFields() { - Stream<ImmutableSDField> extraFields = extraFieldList().stream().map(ImmutableSDField.class::cast); - Stream<ImmutableSDField> documentFields = docType.fieldSet().stream().map(ImmutableSDField.class::cast); - return Stream.concat( - extraFields, - Stream.concat(documentFields, allImportedFields())); + public List<ImmutableSDField> allFieldsList() { + List<ImmutableSDField> all = new ArrayList<>(); + all.addAll(extraFieldList()); + for (Field field : docType.fieldSet()) { + all.add((ImmutableSDField) field); + } + if (importedFields.isPresent()) { + for (ImportedField imported : importedFields.get().fields().values()) { + all.add(imported.asImmutableSDField()); + } + } + return all; } /** @@ -228,6 +237,7 @@ public class Search implements Serializable, ImmutableSearch { * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not * impact this */ + @Override public List<SDField> allConcreteFields() { List<SDField> allFields = new ArrayList<>(); allFields.addAll(extraFieldList()); @@ -240,10 +250,12 @@ public class Search implements Serializable, ImmutableSearch { /** * Returns the content of a ranking expression file */ + @Override public Reader getRankingExpression(String fileName) { return applicationPackage.getRankingExpression(fileName); } + @Override public ApplicationPackage applicationPackage() { return applicationPackage; } /** @@ -253,6 +265,7 @@ public class Search implements Serializable, ImmutableSearch { * @param name of the field * @return the SDField representing the field */ + @Override public SDField getConcreteField(String name) { SDField field = getExtraField(name); if (field != null) { @@ -331,6 +344,7 @@ public class Search implements Serializable, ImmutableSearch { * @param name the name of the index to get * @return the index requested */ + @Override public Index getIndex(String name) { List<Index> sameIndices = new ArrayList<>(1); Index searchIndex = indices.get(name); @@ -338,7 +352,7 @@ public class Search implements Serializable, ImmutableSearch { sameIndices.add(searchIndex); } - for (SDField field : allConcreteFields()) { + for (ImmutableSDField field : allConcreteFields()) { Index index = field.getIndex(name); if (index != null) { sameIndices.add(index); @@ -357,7 +371,7 @@ public class Search implements Serializable, ImmutableSearch { if (indices.get(name) != null) { return true; } - for (SDField field : allConcreteFields()) { + for (ImmutableSDField field : allConcreteFields()) { if (field.existsIndex(name)) { return true; } @@ -405,9 +419,10 @@ public class Search implements Serializable, ImmutableSearch { * * @return The list of explicit defined indexes. */ + @Override public List<Index> getExplicitIndices() { List<Index> allIndices = new ArrayList<>(indices.values()); - for (SDField field : allConcreteFields()) { + for (ImmutableSDField field : allConcreteFields()) { for (Index index : field.getIndices().values()) { allIndices.add(index); } @@ -530,7 +545,7 @@ public class Search implements Serializable, ImmutableSearch { * @return The Attribute with given name. */ public Attribute getAttribute(String name) { - for (SDField field : allConcreteFields()) { + for (ImmutableSDField field : allConcreteFields()) { Attribute attribute = field.getAttributes().get(name); if (attribute != null) { return attribute; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java index be5f135f819..9c183d99435 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java @@ -24,7 +24,7 @@ public class ImmutableImportedSDField implements ImmutableSDField { private final ImportedField importedField; - public ImmutableImportedSDField(ImportedField importedField) { + ImmutableImportedSDField(ImportedField importedField) { this.importedField = importedField; } @@ -63,14 +63,16 @@ public class ImmutableImportedSDField implements ImmutableSDField { } @Override - public ImmutableSDField getBackingField() { return importedField.targetField(); } - - @Override public boolean isIndexStructureField() { return importedField.targetField().isIndexStructureField(); } @Override + public boolean hasIndex() { + return importedField.targetField().hasIndex(); + } + + @Override public boolean usesStructOrMap() { return importedField.targetField().usesStructOrMap(); } @@ -81,6 +83,11 @@ public class ImmutableImportedSDField implements ImmutableSDField { } @Override + public SummaryField getSummaryField(String name) { + return importedField.targetField().getSummaryField(name); + } + + @Override public Index getIndex(String name) { if ( ! importedField.fieldName().equals(name)) { throw new IllegalArgumentException("Getting an index (" + name + ") with different name than the imported field (" @@ -158,6 +165,31 @@ public class ImmutableImportedSDField implements ImmutableSDField { return importedField.fieldName(); // Name of the imported field, not the target field } + @Override + public int getWeight() { + return importedField.targetField().getWeight(); + } + + @Override + public int getLiteralBoost() { + return importedField.targetField().getLiteralBoost(); + } + + @Override + public RankType getRankType() { + return importedField.targetField().getRankType(); + } + + @Override + public Map<String, Index> getIndices() { + return importedField.targetField().getIndices(); + } + + @Override + public boolean existsIndex(String name) { + return importedField.targetField().existsIndex(name); + } + /** * Returns a field representation of the imported field. * Changes to the returned instance are not propagated back to the underlying imported field! diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java index 15e75ad8314..7f92e676118 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java @@ -33,16 +33,6 @@ public interface ImmutableSDField { boolean isImportedField(); - /** - * Returns the field backing this - the field itself if this is a regular field, - * and the target field if this is imported. - */ - ImmutableSDField getBackingField(); - - default boolean isConcreteField() { - return !isImportedField(); - } - boolean isIndexStructureField(); boolean usesStructOrMap(); @@ -83,4 +73,11 @@ public interface ImmutableSDField { Field asField(); boolean hasFullIndexingDocprocRights(); + int getWeight(); + int getLiteralBoost(); + RankType getRankType(); + Map<String, Index> getIndices(); + boolean existsIndex(String name); + SummaryField getSummaryField(String name); + boolean hasIndex(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 8b523211471..c657d29033a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -1,7 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; -import com.yahoo.document.*; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.searchdefinition.Index; @@ -12,12 +17,24 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; -import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; import com.yahoo.vespa.indexinglanguage.parser.ParseException; -import java.io.Serializable; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.TreeMap; + /** * The field class represents a document field. It is used in @@ -28,7 +45,7 @@ import java.util.*; * * @author bratseth */ -public class SDField extends Field implements TypedKey, FieldOperationContainer, ImmutableSDField, Serializable { +public class SDField extends Field implements TypedKey, FieldOperationContainer, ImmutableSDField { /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ private boolean indexStructureField = false; @@ -89,7 +106,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, private Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0); /** The document that this field was declared in, or null*/ - protected SDDocumentType ownerDocType = null; + private SDDocumentType ownerDocType = null; /** The aliases declared for this field. May pertain to indexes or attributes */ private Map<String, String> aliasToName = new HashMap<>(); @@ -202,9 +219,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } @Override - public ImmutableSDField getBackingField() { return this; } - - @Override public boolean doesAttributing() { return containsExpression(AttributeExpression.class); } @@ -235,7 +249,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return findExpression(searchFor) != null; } - public <T extends Expression> T findExpression(Class<T> searchFor) { + private <T extends Expression> T findExpression(Class<T> searchFor) { return new ExpressionSearcher<>(searchFor).searchIn(indexingScript); } @@ -401,7 +415,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return (dataType instanceof StructDataType) ? (StructDataType)dataType : null; } - public DataType getFirstStructOrMapRecursive() { + private DataType getFirstStructOrMapRecursive() { DataType dataType = getDataType(); while (dataType instanceof CollectionDataType) { // Currently no nesting of collections dataType = ((CollectionDataType)dataType).getNestedType(); @@ -409,7 +423,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null; } - public boolean usesStruct() { + private boolean usesStruct() { DataType dt = getFirstStructRecursive(); return (dt != null); } @@ -421,7 +435,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } /** Parse an indexing expression which will use the simple linguistics implementatino suitable for testing */ - @SuppressWarnings("deprecation") public void parseIndexingScript(String script) { parseIndexingScript(script, new SimpleLinguistics()); } @@ -496,20 +509,9 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, this.indexStructureField = indexStructureField; } - /** - * Returns an iterator of the index names this should index to - * (whether set explicitly or not) - */ - public Iterator<String> getFieldNameAsIterator() { // TODO: Replace usage by getName - return Collections.singletonList(getName()).iterator(); - } - - /** Returns 1 if this is indexed, 0 if it is not indexed */ // TODO: Replace by a boolean method, or something, see hasIndex - public int getIndexToCount() { - if (getIndexingScript() == null) return 0; - if (!doesIndexing()) return 0; - - return 1; + @Override + public boolean hasIndex() { + return (getIndexingScript() != null) && doesIndexing(); } /** Sets the literal boost of this field */ @@ -520,12 +522,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * when a query term matched as query term exactly (unnormalized and unstemmed). * Default is non-positive. */ + @Override public int getLiteralBoost() { return literalBoost; } /** Sets the weight of this field */ public void setWeight(int weight) { this.weight=weight; } /** Returns the weight of this field, or 0 if nothing is set */ + @Override public int getWeight() { return weight; } /** @@ -583,20 +587,17 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * Returns an index if this field has one (implicitly or * explicitly) targeting the given name. */ + @Override public boolean existsIndex(String name) { if (indices.get(name) != null) return true; - if (name.equals(getName())) { - if (doesIndexing()) { - return true; - } - } - return false; + return name.equals(getName()) && doesIndexing(); } /** * Defined indices on this field * @return defined indices on this */ + @Override public Map<String, Index> getIndices() { return indices; } @@ -621,6 +622,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, public Ranking getRanking() { return ranking; } /** Returns the default rank type of indices of this field, or null if nothing is set */ + @Override public RankType getRankType() { return this.rankType; } /** @@ -695,6 +697,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * Returns a summary field defined (implicitly or explicitly) by this field. * Returns null if there is no such summary field defined. */ + @Override public SummaryField getSummaryField(String name) { return summaryFields.get(name); } @@ -770,7 +773,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, * The document that this field was declared in, or null * */ - public SDDocumentType getOwnerDocType() { + private SDDocumentType getOwnerDocType() { return ownerDocType; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index c76b8536ea0..a12b06624cf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -4,8 +4,10 @@ package com.yahoo.searchdefinition.expressiontransforms; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.transform.TransformContext; +import com.yahoo.tensor.TensorType; import java.util.HashMap; import java.util.Map; @@ -25,10 +27,11 @@ public class RankProfileTransformContext extends TransformContext { public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, + Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, Map<String, Value> constants, Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) { - super(constants, rankProfile.typeContext(queryProfiles)); + super(constants, rankProfile.typeContext(queryProfiles, featureTypes)); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; this.importedModels = importedModels; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index e75547a5bb2..d1eb18c4916 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -47,7 +47,7 @@ public class AddExtraFieldsToDocument extends Processor { } private void addSdField(Search search, SDDocumentType document, SDField field, boolean validate) { - if (field.getIndexToCount() == 0 && field.getAttributes().isEmpty()) { + if (! field.hasIndex() && field.getAttributes().isEmpty()) { return; } for (Attribute atr : field.getAttributes().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java index 94589d94255..5bcb2ddf54f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.Search; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -21,7 +22,7 @@ public class AttributeProperties extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - for (SDField field : search.allConcreteFields()) { + for (ImmutableSDField field : search.allConcreteFields()) { String fieldName = field.getName(); // For each attribute, check if the attribute has been created @@ -56,7 +57,7 @@ public class AttributeProperties extends Processor { * @param attributeName name of the attribute * @return true if the attribute has been created by this field, else false */ - static boolean attributeCreated(SDField field, String attributeName) { + static boolean attributeCreated(ImmutableSDField field, String attributeName) { if ( ! field.doesAttributing()) { return false; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index 23257e5eafd..55f101a4877 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -4,9 +4,9 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.document.NumericDataType; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.Search; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -24,20 +24,20 @@ public class AttributesImplicitWord extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - for (SDField field : search.allConcreteFields()) { + for (ImmutableSDField field : search.allConcreteFields()) { if (fieldImplicitlyWordMatch(field)) { field.getMatching().setType(Matching.Type.WORD); } } } - private boolean fieldImplicitlyWordMatch(SDField field) { + private boolean fieldImplicitlyWordMatch(ImmutableSDField field) { // numeric types should not trigger exact-match query parsing DataType dt = field.getDataType().getPrimitiveType(); if (dt != null && dt instanceof NumericDataType) { return false; } - return (field.getIndexToCount() == 0 + return (! field.hasIndex() && !field.getAttributes().isEmpty() && field.getIndices().isEmpty() && !field.getMatching().isTypeUserSet()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java index b59d3527e87..86a272d5201 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.Search; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -24,7 +24,7 @@ public class Bolding extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - for (SDField field : search.allConcreteFields()) { + for (ImmutableSDField field : search.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields().values()) { if (summary.getTransform().isBolded() && !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 1994b1096ce..267ff827e77 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; +import com.yahoo.document.Field; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; @@ -34,7 +35,7 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { } } - private void checkFieldType(SDField field, DataType dataType) { + private void checkFieldType(Field field, DataType dataType) { if (dataType instanceof ArrayDataType) { DataType nestedType = ((ArrayDataType) dataType).getNestedType(); checkFieldType(field, nestedType); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index a871da20669..51751b2e247 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -2,14 +2,20 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.*; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; -import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.expressions.ExactExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -21,7 +27,7 @@ public class ExactMatch extends Processor { public static final String DEFAULT_EXACT_TERMINATOR = "@@"; - public ExactMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + ExactMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java deleted file mode 100644 index 41355a76f47..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Search; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Iterator; - -/** - * Fail if: - * 1) There are index: settings without explicit index names (name same as field name) - * 2) All the index-to indexes differ from the field name. - * - * @author Vegard Havdal - */ -public class IndexSettingsNonFieldNames extends Processor { - - public IndexSettingsNonFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : search.allConcreteFields()) { - boolean fieldNameUsed = false; - for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) { - String iName = (String)(i.next()); - if (iName.equals(field.getName())) { - fieldNameUsed = true; - } - } - if ( ! fieldNameUsed) { - for (Index index : field.getIndices().values()) { - if (index.getName().equals(field.getName())) { - throw new IllegalArgumentException("Error in " + field + " in " + search + - ": When all index names differ from field name, index " + - "parameter settings must specify index name explicitly."); - } - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index 27520647e3b..dedc96cdc05 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -2,14 +2,27 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.*; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldTypeAdapter; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.HashSet; @@ -20,7 +33,7 @@ import java.util.Set; */ public class IndexingValidation extends Processor { - public IndexingValidation(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + IndexingValidation(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @@ -83,7 +96,7 @@ public class IndexingValidation extends Processor { final Search search; - public MyAdapter(Search search) { + MyAdapter(Search search) { this.search = search; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java index a52a8ab74e6..7adb7f44594 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -10,7 +10,6 @@ import com.yahoo.searchdefinition.processing.multifieldresolver.RankTypeResolver import com.yahoo.searchdefinition.processing.multifieldresolver.StemmingResolver; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,11 +38,7 @@ public class MultifieldIndexHarmonizer extends Processor { private void populateIndexToFields(Search search) { for (SDField field : search.allConcreteFields() ) { if ( ! field.doesIndexing()) continue; - - for (Iterator j = field.getFieldNameAsIterator(); j.hasNext();) { - String indexName = (String)j.next(); - addIndexField(indexName, field); - } + addIndexField(field.getName(), field); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index b0ba8e30f06..3f225b00277 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -52,7 +52,6 @@ public class Processing { Bolding::new, AttributeProperties::new, SetRankTypeEmptyOnFilters::new, - IndexSettingsNonFieldNames::new, SummaryDynamicStructsArrays::new, StringSettingsOnNonStringFields::new, IndexingOutputs::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index 6bfd0ef29ea..e15e17817a2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -2,7 +2,8 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.document.*; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; @@ -79,12 +80,10 @@ public abstract class Processor { implementationField.setStemming(Stemming.NONE); implementationField.getNormalizing().inferCodepoint(); implementationField.parseIndexingScript(indexing); - for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) { - String indexName = (String)i.next(); - String implementationIndexName = indexName + "_" + suffix; - Index implementationIndex = new Index(implementationIndexName); - search.addIndex(implementationIndex); - } + String indexName = field.getName(); + String implementationIndexName = indexName + "_" + suffix; + Index implementationIndex = new Index(implementationIndexName); + search.addIndex(implementationIndex); if (queryCommand != null) { field.addQueryCommand(queryCommand); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index d81fdf70d20..d0a0bbfb748 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -2,8 +2,11 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.*; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -20,7 +23,7 @@ public class UriHack extends Processor { private static final List<String> URL_SUFFIX = Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); - public UriHack(Search search, + UriHack(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java index c6b83349691..2a86247a973 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -21,7 +21,7 @@ public class UrlFieldValidator extends Processor { public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - for (SDField field : search.allConcreteFields()) { + for (ImmutableSDField field : search.allConcreteFields()) { if ( ! field.getDataType().equals(DataType.URI)) continue; if (field.doesAttributing()) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index 408d60e1cff..9040cffc81e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.Ranking; import com.yahoo.searchdefinition.document.SDField; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 3bde76c1c79..58ef47b7ba9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -14,7 +14,6 @@ import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.ImportedField; import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.processing.Processor; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -45,7 +44,7 @@ public class RankProfileTypeSettingsProcessor extends Processor { private void processAttributeFields() { if (search == null) return; // we're processing global profiles - for (SDField field : search.allConcreteFields()) { + for (ImmutableSDField field : search.allConcreteFields()) { Attribute attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.tensorType().isPresent()) { addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index 6a94c04759f..a3d2cbcebeb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.DerivedConfiguration; @@ -29,7 +29,7 @@ public class NoPrefixForIndexes extends Validator { for (DocumentDatabase docDb : sc.getDocumentDbs()) { DerivedConfiguration sdConfig = docDb.getDerivedConfiguration(); Search search = sdConfig.getSearch(); - for (SDField field : search.allConcreteFields()) { + for (ImmutableSDField field : search.allConcreteFields()) { if (field.doesIndexing()) { //if (!field.getIndexTo().isEmpty() && !field.getIndexTo().contains(field.getName())) continue; if (field.getMatching().getAlgorithm().equals(Matching.Algorithm.PREFIX)) { @@ -47,7 +47,7 @@ public class NoPrefixForIndexes extends Validator { } } - private void failField(Search search, SDField field) { + private void failField(Search search, ImmutableSDField field) { throw new IllegalArgumentException("For search '" + search.getName() + "', field '" + field.getName() + "': match/index:prefix is not supported for indexes."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index e75791906bd..39478730982 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -7,8 +7,8 @@ import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; @@ -38,7 +38,7 @@ public class StreamingValidator extends Validator { private static void warnStreamingGramMatching(SearchCluster sc, DeployLogger logger) { if (sc.getSdConfig() != null) { - for (SDField sd : sc.getSdConfig().getSearch().allConcreteFields()) { + for (ImmutableSDField sd : sc.getSdConfig().getSearch().allConcreteFields()) { if (sd.getMatching().getType().equals(Matching.Type.GRAM)) { logger.log(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + "', SD field '" + sd.getName() + "': n-gram matching is not supported for streaming search."); @@ -55,7 +55,7 @@ public class StreamingValidator extends Validator { */ private static void warnStreamingAttributes(SearchCluster sc, DeployLogger logger) { if (sc.getSdConfig() != null) { - for (SDField sd : sc.getSdConfig().getSearch().allConcreteFields()) { + for (ImmutableSDField sd : sc.getSdConfig().getSearch().allConcreteFields()) { if (sd.doesAttributing()) { warnStreamingAttribute(sc, sd, logger); } @@ -63,7 +63,7 @@ public class StreamingValidator extends Validator { } } - private static void warnStreamingAttribute(SearchCluster sc, SDField sd, DeployLogger logger) { + private static void warnStreamingAttribute(SearchCluster sc, ImmutableSDField sd, DeployLogger logger) { // If the field is numeric, we can't print this, because we may have converted the field to // attribute indexing ourselves (IntegerIndex2Attribute) if (sd.getDataType() instanceof NumericDataType) return; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index d680f6bd37c..049e71be23c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -19,12 +19,12 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; public class IndexingScriptChangeMessageBuilder { private final Search currentSearch; - private final SDField currentField; + private final ImmutableSDField currentField; private final Search nextSearch; - private final SDField nextField; + private final ImmutableSDField nextField; - public IndexingScriptChangeMessageBuilder(Search currentSearch, SDField currentField, - Search nextSearch, SDField nextField) { + public IndexingScriptChangeMessageBuilder(Search currentSearch, ImmutableSDField currentField, + Search nextSearch, ImmutableSDField nextField) { this.currentSearch = currentSearch; this.currentField = currentField; this.nextSearch = nextSearch; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index ff9230b34f3..b03141fa5d9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; @@ -35,9 +35,9 @@ public class IndexingScriptChangeValidator { public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { List<VespaConfigChangeAction> result = new ArrayList<>(); - for (SDField nextField : nextSearch.allConcreteFields()) { + for (ImmutableSDField nextField : nextSearch.allConcreteFields()) { String fieldName = nextField.getName(); - SDField currentField = currentSearch.getConcreteField(fieldName); + ImmutableSDField currentField = currentSearch.getConcreteField(fieldName); if (currentField != null) { validateScripts(currentField, nextField, overrides, now).ifPresent(r -> result.add(r)); } @@ -45,7 +45,7 @@ public class IndexingScriptChangeValidator { return result; } - private Optional<VespaConfigChangeAction> validateScripts(SDField currentField, SDField nextField, + private Optional<VespaConfigChangeAction> validateScripts(ImmutableSDField currentField, ImmutableSDField nextField, ValidationOverrides overrides, Instant now) { ScriptExpression currentScript = currentField.getIndexingScript(); ScriptExpression nextScript = nextField.getIndexingScript(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java index c5189069fca..d82c4bbaa7f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java @@ -3,12 +3,12 @@ package com.yahoo.searchdefinition; import static org.junit.Assert.*; +import com.yahoo.searchdefinition.document.ImmutableSDField; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.parser.ParseException; /** @@ -65,7 +65,7 @@ public class PredicateDataTypeTestCase { upperBoundParameter(upperBound)))); SearchBuilder sb = SearchBuilder.createFromString(sd); - for (SDField field : sb.getSearch().allConcreteFields()) { + for (ImmutableSDField field : sb.getSearch().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { assertTrue(index.getBooleanIndexDefiniton().hasArity()); @@ -92,7 +92,7 @@ public class PredicateDataTypeTestCase { upperBoundParameter(upperBound)))); SearchBuilder sb = SearchBuilder.createFromString(sd); - for (SDField field : sb.getSearch().allConcreteFields()) { + for (ImmutableSDField field : sb.getSearch().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { assertEquals(arity, index.getBooleanIndexDefiniton().getArity()); @@ -110,7 +110,7 @@ public class PredicateDataTypeTestCase { attributeFieldSd( arityParameter(2)))); SearchBuilder sb = SearchBuilder.createFromString(sd); - for (SDField field : sb.getSearch().allConcreteFields()) { + for (ImmutableSDField field : sb.getSearch().allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { for (Index index : field.getIndices().values()) { assertTrue(index.getBooleanIndexDefiniton().hasArity()); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index 74a14e51122..009d8fd73cb 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -2,7 +2,6 @@ package com.yahoo.config.provision; import java.util.List; -import java.util.Set; /** * The possible types of nodes in the node repository @@ -36,13 +35,13 @@ public enum NodeType { controllerhost("Controller host", controller), /** A host of multiple nodes, only used in {@link SystemName#dev} */ - devhost("Dev host", tenant, config, controller); + devhost("Dev host", config, controller, tenant); private final List<NodeType> childNodeTypes; private final String description; NodeType(String description, NodeType... childNodeTypes) { - this.childNodeTypes = List.copyOf(Set.of(childNodeTypes)); + this.childNodeTypes = List.of(childNodeTypes); this.description = description; } diff --git a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp index 432fe8c1afb..a5340937713 100644 --- a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp +++ b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp @@ -116,7 +116,12 @@ PingProxy::Main() if (debugging) { printf("connecting to '%s'\n", spec); } - initRPC(spec); + try { + initRPC(spec); + } catch (std::exception& ex) { + LOG(error, "Got exception while initializing RPC: '%s'", ex.what()); + return 1; + } FRT_RPCRequest *req = _server->supervisor().AllocRPCRequest(); diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index bd929cb17d4..7a06b13d435 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -24,6 +24,3 @@ athenzCaTrustStore string # Period between certificate updates updatePeriodDays int default=1 - -# Tenant Service id -tenantService string default=vespa.vespa.tenant
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index 694c30eba9a..badb99c0523 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -17,7 +17,6 @@ import com.yahoo.search.searchchain.PhaseNames; import java.util.Iterator; import java.util.Map; - /** * A searcher which does parametrized collapsing. * @@ -29,9 +28,9 @@ import java.util.Map; public class FieldCollapsingSearcher extends Searcher { private static final CompoundName collapse = new CompoundName("collapse"); - private static final CompoundName collapsefield=new CompoundName("collapsefield"); - private static final CompoundName collapsesize=new CompoundName("collapsesize"); - private static final CompoundName collapseSummaryName=new CompoundName("collapse.summary"); + private static final CompoundName collapsefield = new CompoundName("collapsefield"); + private static final CompoundName collapsesize = new CompoundName("collapsesize"); + private static final CompoundName collapseSummaryName = new CompoundName("collapse.summary"); /** Maximum number of queries to send next searcher */ private int maxQueries = 4; @@ -64,6 +63,7 @@ public class FieldCollapsingSearcher extends Searcher { } @Inject + @SuppressWarnings("unused") public FieldCollapsingSearcher(QrSearchersConfig config) { QrSearchersConfig.Com.Yahoo.Prelude.Searcher.FieldCollapsingSearcher s = config.com().yahoo().prelude().searcher().FieldCollapsingSearcher(); @@ -99,7 +99,7 @@ public class FieldCollapsingSearcher extends Searcher { public Result search(com.yahoo.search.Query query, Execution execution) { String collapseField = query.properties().getString(collapsefield); - if (collapseField==null) return execution.search(query); + if (collapseField == null) return execution.search(query); int collapseSize = query.properties().getInteger(collapsesize,defaultCollapseSize); query.properties().set(collapse, "0"); @@ -113,11 +113,12 @@ public class FieldCollapsingSearcher extends Searcher { int performedQueries = 0; Result resultSource; String collapseSummary = query.properties().getString(collapseSummaryName); + String summaryClass = (collapseSummary == null) + ? query.getPresentation().getSummary() : collapseSummary; + query.trace("Collapsing by '" + collapseField + "' using summary '" + collapseSummary + "'", 2); do { resultSource = search(query.clone(), execution, nextOffset, hitsToRequest); - String summaryClass = (collapseSummary == null) - ? query.getPresentation().getSummary() : collapseSummary; fill(resultSource, summaryClass, execution); collapse(result, knownCollapses, resultSource, collapseField, collapseSize); @@ -146,7 +147,7 @@ public class FieldCollapsingSearcher extends Searcher { return result; } - private Result search(Query query, Execution execution, int offset , int hits) { + private Result search(Query query, Execution execution, int offset, int hits) { query.setOffset(offset); query.setHits(hits); return execution.search(query); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 8c6c4e31808..84565472820 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -312,6 +312,16 @@ public class QueryTestCase { } @Test + public void testQueryProfileSourceAccess() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("myField", "Profile: %{queryProfile}", null); + Query query = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + + String source = query.properties().getInstance(com.yahoo.search.query.profile.QueryProfileProperties.class).getQueryProfile().listValuesWithSources(new CompoundName(""), query.getHttpRequest().propertyMap(), query.properties()).get("myField").source(); + assertEquals("myProfile", source); + } + + @Test public void testBooleanParameter() { QueryProfile profile = new QueryProfile("myProfile"); Query query = new Query("/?query=something&ranking.softtimeout.enable=false", profile.compile(null)); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 26f78a3be90..578be18c4d7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -34,7 +34,7 @@ public interface ServiceRegistry { ConfigServer configServer(); - Clock clock(); + default Clock clock() { return Clock.systemUTC(); } NameService nameService(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 89f0a3c3382..f8b4d4171a4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -19,6 +19,10 @@ import java.util.Set; */ enum PathGroup { + /** Paths exclusive to operators (including read), used for system management. */ + classifiedOperator(Optional.of("/api"), + "/configserver/v1/{*}"), + /** Paths used for system management by operators. */ operator("/controller/v1/{*}", "/flags/v1/{*}", @@ -228,6 +232,10 @@ enum PathGroup { return EnumSet.allOf(PathGroup.class); } + static Set<PathGroup> allExcept(PathGroup... pathGroups) { + return EnumSet.complementOf(EnumSet.copyOf(List.of(pathGroups))); + } + /** Returns whether this group matches path in given context */ boolean matches(URI uri, Context context) { return get(uri).map(p -> { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index db7dd5909b3..e0341d76950 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -115,7 +115,7 @@ enum Policy { /** Read access to all information in select systems. */ classifiedRead(Privilege.grant(Action.read) - .on(PathGroup.all()) + .on(PathGroup.allExcept(PathGroup.classifiedOperator)) .in(SystemName.main, SystemName.cd, SystemName.dev)), /** Read access to public info. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 72b11f56d9f..dd43195f67d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,19 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; +import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; import org.apache.http.client.config.RequestConfig; @@ -24,20 +18,17 @@ import org.apache.http.client.methods.HttpPatch; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.conn.ssl.X509HostnameVerifier; import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; -import javax.net.ssl.SSLException; +import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLSession; -import javax.net.ssl.SSLSocket; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; import java.net.URI; -import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -45,44 +36,45 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.yahoo.yolean.Exceptions.uncheck; -import static java.util.Collections.singleton; /** * @author Haakon Dybdahl * @author bjorncs */ @SuppressWarnings("unused") // Injected -public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { +public class ConfigServerRestExecutorImpl extends AbstractComponent implements ConfigServerRestExecutor { private static final Logger log = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10); private static final Set<String> HEADERS_TO_COPY = Set.of("X-HTTP-Method-Override", "Content-Type"); - private final ZoneRegistry zoneRegistry; - private final ServiceIdentityProvider sslContextProvider; + private final CloseableHttpClient client; @Inject - public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, - ServiceIdentityProvider sslContextProvider) { - this.zoneRegistry = zoneRegistry; - this.sslContextProvider = sslContextProvider; + public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) { + RequestConfig config = RequestConfig.custom() + .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); + + this.client = createHttpClient(config, sslContextProvider, + new ControllerOrConfigserverHostnameVerifier(zoneRegistry)); } @Override public ProxyResponse handle(ProxyRequest proxyRequest) throws ProxyException { - if (proxyRequest.isDiscoveryRequest()) { - return createDiscoveryResponse(proxyRequest); - } - - ZoneId zoneId = ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion()); - - List<URI> allServers = getConfigserverEndpoints(zoneId); + // Make a local copy of the list as we want to manipulate it in case of ping problems. + List<URI> allServers = new ArrayList<>(proxyRequest.getTargets()); StringBuilder errorBuilder = new StringBuilder(); - if (queueFirstServerIfDown(allServers, proxyRequest)) { + if (queueFirstServerIfDown(allServers)) { errorBuilder.append("Change ordering due to failed ping."); } for (URI uri : allServers) { @@ -96,60 +88,14 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { + errorBuilder.toString())); } - private List<URI> getConfigserverEndpoints(ZoneId zoneId) { - // TODO: Use config server VIP for all zones that have one - // Make a local copy of the list as we want to manipulate it in case of ping problems. - if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { - return Collections.singletonList(zoneRegistry.getConfigServerVipUri(zoneId)); - } else { - return new ArrayList<>(zoneRegistry.getConfigServerUris(zoneId)); - } - } - - private static class DiscoveryResponseStructure { - public List<String> uris = new ArrayList<>(); - } - - private ProxyResponse createDiscoveryResponse(ProxyRequest proxyRequest) { - ObjectMapper mapper = new ObjectMapper(); - DiscoveryResponseStructure responseStructure = new DiscoveryResponseStructure(); - String environmentName = proxyRequest.getEnvironment(); - - ZoneList zones = zoneRegistry.zones().all(); - if ( ! environmentName.isEmpty()) - zones = zones.in(Environment.from(environmentName)); - - for (ZoneApi zone : zones.zones()) { - responseStructure.uris.add(proxyRequest.getScheme() + "://" + proxyRequest.getControllerPrefix() + - zone.getEnvironment().value() + "/" + zone.getRegionName().value()); - } - JsonNode node = mapper.valueToTree(responseStructure); - return new ProxyResponse(proxyRequest, node.toString(), 200, Optional.empty(), "application/json"); - } - - private static String removeFirstSlashIfAny(String url) { - if (url.startsWith("/")) { - return url.substring(1); - } - return url; - } - private Optional<ProxyResponse> proxyCall(URI uri, ProxyRequest proxyRequest, StringBuilder errorBuilder) throws ProxyException { - String fullUri = uri.toString() + removeFirstSlashIfAny(proxyRequest.getConfigServerRequest()); final HttpRequestBase requestBase = createHttpBaseRequest( - proxyRequest.getMethod(), fullUri, proxyRequest.getData()); + proxyRequest.getMethod(), proxyRequest.createConfigServerRequestUri(uri), proxyRequest.getData()); // Empty list of headers to copy for now, add headers when needed, or rewrite logic. copyHeaders(proxyRequest.getHeaders(), requestBase); - RequestConfig config = RequestConfig.custom() - .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); - try ( - CloseableHttpClient client = createHttpClient(config, sslContextProvider, zoneRegistry, proxyRequest); - CloseableHttpResponse response = client.execute(requestBase) - ) { + try (CloseableHttpResponse response = client.execute(requestBase)) { String content = getContent(response); int status = response.getStatusLine().getStatusCode(); if (status / 100 == 5) { @@ -168,7 +114,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { contentType = "application/json"; } // Send response back - return Optional.of(new ProxyResponse(proxyRequest, content, status, Optional.of(uri), contentType)); + return Optional.of(new ProxyResponse(proxyRequest, content, status, uri, contentType)); } catch (Exception e) { errorBuilder.append("Talking to server ").append(uri.getHost()); errorBuilder.append(" got exception ").append(e.getMessage()); @@ -179,20 +125,12 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { private static String getContent(CloseableHttpResponse response) { return Optional.ofNullable(response.getEntity()) - .map(entity -> - { - try { - return EntityUtils.toString(entity); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - ).orElse(""); + .map(entity -> uncheck(() -> EntityUtils.toString(entity))) + .orElse(""); } - private static HttpRequestBase createHttpBaseRequest(String method, String uri, InputStream data) throws ProxyException { - Method enumMethod = Method.valueOf(method); - switch (enumMethod) { + private static HttpRequestBase createHttpBaseRequest(Method method, URI uri, InputStream data) throws ProxyException { + switch (method) { case GET: return new HttpGet(uri); case POST: @@ -235,7 +173,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { * if it is not responding, we try the other servers first. False positive/negatives are not critical, * but will increase latency to some extent. */ - private boolean queueFirstServerIfDown(List<URI> allServers, ProxyRequest proxyRequest) { + private boolean queueFirstServerIfDown(List<URI> allServers) { if (allServers.size() < 2) { return false; } @@ -247,11 +185,8 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { .setConnectTimeout(timeout) .setConnectionRequestTimeout(timeout) .setSocketTimeout(timeout).build(); - try ( - CloseableHttpClient client = createHttpClient(config, sslContextProvider, zoneRegistry, proxyRequest); - CloseableHttpResponse response = client.execute(httpget) - - ) { + httpget.setConfig(config); + try (CloseableHttpResponse response = client.execute(httpget)) { if (response.getStatusLine().getStatusCode() == 200) { return false; } @@ -260,61 +195,51 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { // We ignore this, if server is restarting this might happen. } // Some error happened, move this server to the back. The other servers should be running. - allServers.remove(0); - allServers.add(uri); + Collections.rotate(allServers, -1); return true; } - @SuppressWarnings("deprecation") + @Override + public void deconstruct() { + try { + client.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private static CloseableHttpClient createHttpClient(RequestConfig config, ServiceIdentityProvider sslContextProvider, - ZoneRegistry zoneRegistry, - ProxyRequest proxyRequest) { - AthenzIdentityVerifier hostnameVerifier = - new AthenzIdentityVerifier( - singleton( - zoneRegistry.getConfigServerHttpsIdentity( - ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); + HostnameVerifier hostnameVerifier) { return HttpClientBuilder.create() .setUserAgent("config-server-proxy-client") .setSslcontext(sslContextProvider.getIdentitySslContext()) - .setHostnameVerifier(new AthenzIdentityVerifierAdapter(hostnameVerifier)) + .setSSLHostnameVerifier(hostnameVerifier) .setDefaultRequestConfig(config) + .setMaxConnPerRoute(10) + .setMaxConnTotal(500) + .setConnectionTimeToLive(1, TimeUnit.MINUTES) .build(); } - @SuppressWarnings("deprecation") - private static class AthenzIdentityVerifierAdapter implements X509HostnameVerifier { + private static class ControllerOrConfigserverHostnameVerifier implements HostnameVerifier { - private final AthenzIdentityVerifier verifier; + private final HostnameVerifier configserverVerifier; - AthenzIdentityVerifierAdapter(AthenzIdentityVerifier verifier) { - this.verifier = verifier; + ControllerOrConfigserverHostnameVerifier(ZoneRegistry registry) { + this.configserverVerifier = createConfigserverVerifier(registry); } - @Override - public boolean verify(String hostname, SSLSession sslSession) { - return verifier.verify(hostname, sslSession); - } - - @Override - public void verify(String host, SSLSocket ssl) { /* All sockets accepted */} - - @Override - public void verify(String hostname, X509Certificate certificate) throws SSLException { - AthenzIdentity identity = AthenzIdentities.from(certificate); - if (!verifier.isTrusted(identity)) { - throw new SSLException("Athenz identity is not trusted: " + identity.getFullName()); - } + private static HostnameVerifier createConfigserverVerifier(ZoneRegistry registry) { + Set<AthenzIdentity> configserverIdentities = registry.zones().all().zones().stream() + .map(zone -> registry.getConfigServerHttpsIdentity(zone.getId())) + .collect(Collectors.toSet()); + return new AthenzIdentityVerifier(configserverIdentities); } @Override - public void verify(String hostname, String[] cns, String[] subjectAlts) throws SSLException { - AthenzIdentity identity = AthenzIdentities.from(cns[0]); - if (!verifier.isTrusted(identity)) { - throw new SSLException("Athenz identity is not trusted: " + identity.getFullName()); - } + public boolean verify(String hostname, SSLSession session) { + return "localhost".equals(hostname) || configserverVerifier.verify(hostname, session); } } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index 6854d583222..f398683567b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -2,14 +2,15 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.net.HostName; -import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.net.URLDecoder; +import java.net.URISyntaxException; import java.util.List; import java.util.Map; +import java.util.Objects; + +import static com.yahoo.jdisc.http.HttpRequest.Method; /** * Keeping information about the calls that are being proxied. @@ -19,101 +20,84 @@ import java.util.Map; */ public class ProxyRequest { - private final String environment; - private final String region; - private final String configServerRequest; - private final InputStream requestData; + private final Method method; + private final URI requestUri; private final Map<String, List<String>> headers; - private final String method; - private final String controllerPrefix; - private final String scheme; + private final InputStream requestData; + + private final List<URI> targets; + private final String targetPath; /** * The constructor calls exception if the request is invalid. * * @param request the request from the jdisc framework. - * @param pathPrefix the path prefix of the proxy. + * @param targets list of targets this request should be proxied to (targets are tried once in order until a response is returned). + * @param targetPath the path to proxy to. * @throws ProxyException on errors */ - public ProxyRequest(HttpRequest request, String pathPrefix) throws ProxyException, IOException { - this(request.getUri(), request.getJDiscRequest().headers(), request.getData(), request.getMethod().name(), - pathPrefix); + public ProxyRequest(HttpRequest request, List<URI> targets, String targetPath) throws ProxyException { + this(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), request.getData(), + targets, targetPath); } - ProxyRequest(URI requestUri, Map<String, List<String>> headers, InputStream body, String method, - String pathPrefix) throws ProxyException, IOException { - if (requestUri == null) { - throw new ProxyException(ErrorResponse.badRequest("Request not set.")); - } - final String path = URLDecoder.decode(requestUri.getPath(),"UTF-8"); - if (! path.startsWith(pathPrefix)) { - // This has to be caused by wrong mapping of path in services.xml. - throw new ProxyException(ErrorResponse.notFoundError("Request not starting with " + pathPrefix)); - } - final String uriNoPrefix = path.replaceFirst(pathPrefix, "") - + (requestUri.getRawQuery() == null ? "" : "?" + requestUri.getRawQuery()); - - final String[] parts = uriNoPrefix.split("/"); + ProxyRequest(Method method, URI requestUri, Map<String, List<String>> headers, InputStream body, + List<URI> targets, String targetPath) throws ProxyException { + Objects.requireNonNull(requestUri, "Request must be non-null"); + if (!requestUri.getPath().endsWith(targetPath)) + throw new ProxyException(ErrorResponse.badRequest(String.format( + "Request path '%s' does not end with proxy path '%s'", requestUri.getPath(), targetPath))); - this.environment = parts.length > 0 ? parts[0] : ""; - this.region = parts.length > 1 ? parts[1] : ""; - this.configServerRequest = parts.length > 2 ? uriNoPrefix.replace(environment + "/" + region, "") : ""; + this.method = Objects.requireNonNull(method); + this.requestUri = Objects.requireNonNull(requestUri); + this.headers = Objects.requireNonNull(headers); this.requestData = body; - this.headers = headers; - this.method = method; - - String hostPort = headers.containsKey("host") - ? headers.get("host").get(0) - : HostName.getLocalhost() + ":" + requestUri.getPort(); - StringBuilder prefix = new StringBuilder(hostPort + pathPrefix); - if (! environment.isEmpty()) { - prefix.append(environment).append("/").append(region); - } - - this.controllerPrefix = prefix.toString(); - this.scheme = requestUri.getScheme(); - } - /** - * A discovery query lists environments and regions. - */ - public boolean isDiscoveryRequest() { - return region.isEmpty(); + this.targets = List.copyOf(targets); + this.targetPath = targetPath.startsWith("/") ? targetPath : "/" + targetPath; } - public String getRegion() { - return region; - } - public String getEnvironment() { - return environment; + public Method getMethod() { + return method; } - public String getConfigServerRequest() { - return configServerRequest; + public Map<String, List<String>> getHeaders() { + return headers; } public InputStream getData() { return requestData; } - @Override - public String toString() { - return "[ region: " + region + " env: " + environment + " request: " + configServerRequest + "]"; + public List<URI> getTargets() { + return targets; } - public Map<String, List<String>> getHeaders() { - return headers; + public URI createConfigServerRequestUri(URI baseURI) { + try { + return new URI(baseURI.getScheme(), baseURI.getUserInfo(), baseURI.getHost(), + baseURI.getPort(), targetPath, requestUri.getQuery(), requestUri.getFragment()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } - public String getMethod() { - return method; + public URI getControllerPrefixUri() { + String prefixPath = targetPath.equals("/") && !requestUri.getPath().endsWith("/") ? + requestUri.getPath() + targetPath : + requestUri.getPath().substring(0, requestUri.getPath().length() - targetPath.length() + 1); + try { + return new URI(requestUri.getScheme(), requestUri.getUserInfo(), requestUri.getHost(), + requestUri.getPort(), prefixPath, null, null); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } - public String getControllerPrefix() { - return controllerPrefix; + @Override + public String toString() { + return "[targets: " + targets + " request: " + targetPath + "]"; } - public String getScheme() { return scheme; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 3f878740ff0..b0b4f1a556a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -9,14 +9,13 @@ import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.util.Optional; /** * Response class that also rewrites URL from config server. * * @author Haakon Dybdahl */ -public class ProxyResponse extends HttpResponse { +public class ProxyResponse extends HttpResponse { private final String bodyResponseRewritten; private final String contentType; @@ -25,29 +24,21 @@ public class ProxyResponse extends HttpResponse { ProxyRequest controllerRequest, String bodyResponse, int statusResponse, - Optional<URI> configServer, + URI configServer, String contentType) { super(statusResponse); this.contentType = contentType; - if (! configServer.isPresent() || controllerRequest.getControllerPrefix().isEmpty()) { - bodyResponseRewritten = bodyResponse; - return; - } - final String configServerPrefix; final String controllerRequestPrefix; try { configServerPrefix = new URIBuilder() - .setScheme(configServer.get().getScheme()) - .setHost(configServer.get().getHost()) - .setPort(configServer.get().getPort()) - .build().toString(); - controllerRequestPrefix = new URIBuilder() - .setScheme(controllerRequest.getScheme()) - // controller prefix is more than host, so it is a bit hackish, but verified by tests. - .setHost(controllerRequest.getControllerPrefix()) + .setScheme(configServer.getScheme()) + .setHost(configServer.getHost()) + .setPort(configServer.getPort()) + .setPath("/") .build().toString(); + controllerRequestPrefix = controllerRequest.getControllerPrefixUri().toString(); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java new file mode 100644 index 00000000000..64a32bce3c0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -0,0 +1,128 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.configserver; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; +import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; +import com.yahoo.vespa.hosted.controller.proxy.ProxyException; +import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.yolean.Exceptions; + +import java.net.URI; +import java.util.List; +import java.util.logging.Level; +import java.util.stream.Stream; + +/** + * REST API for proxying operator APIs to config servers in a given zone. + * + * @author freva + */ +@SuppressWarnings("unused") +public class ConfigServerApiHandler extends AuditLoggingRequestHandler { + + private static final ZoneId CONTROLLER_ZONE = ZoneId.from("prod", "controller"); + private static final URI CONTROLLER_URI = URI.create("https://localhost:4443/"); + private static final String OPTIONAL_PREFIX = "/api"; + private static final List<String> WHITELISTED_APIS = List.of("/flags/v1/", "/nodes/v2/", "/orchestrator/v1/"); + + private final ZoneRegistry zoneRegistry; + private final ConfigServerRestExecutor proxy; + + public ConfigServerApiHandler(Context parentCtx, ZoneRegistry zoneRegistry, + ConfigServerRestExecutor proxy, Controller controller) { + super(parentCtx, controller.auditLogger()); + this.zoneRegistry = zoneRegistry; + this.proxy = proxy; + } + + @Override + public HttpResponse auditAndHandle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: + return get(request); + case POST: + case PUT: + case DELETE: + case PATCH: + return proxy(request); + default: + return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + } + } catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "', " + + Exceptions.toMessageString(e)); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse get(HttpRequest request) { + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + if (path.matches("/configserver/v1")) { + return root(request); + } + return proxy(request); + } + + private HttpResponse proxy(HttpRequest request) { + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + if ( ! path.matches("/configserver/v1/{environment}/{region}/{*}")) { + return ErrorResponse.notFoundError("Nothing at " + path); + } + + ZoneId zoneId = ZoneId.from(path.get("environment"), path.get("region")); + if (! zoneRegistry.hasZone(zoneId) && ! CONTROLLER_ZONE.equals(zoneId)) { + throw new IllegalArgumentException("No such zone: " + zoneId.value()); + } + + String cfgPath = "/" + path.getRest(); + if (WHITELISTED_APIS.stream().noneMatch(cfgPath::startsWith)) { + return ErrorResponse.forbidden("Cannot access '" + cfgPath + + "' through /configserver/v1, following APIs are permitted: " + String.join(", ", WHITELISTED_APIS)); + } + + try { + return proxy.handle(new ProxyRequest(request, List.of(getEndpoint(zoneId)), cfgPath)); + } catch (ProxyException e) { + throw new RuntimeException(e); + } + } + + private HttpResponse root(HttpRequest request) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + ZoneList zoneList = zoneRegistry.zones().reachable(); + + Cursor zones = root.setArray("zones"); + Stream.concat(Stream.of(CONTROLLER_ZONE), zoneRegistry.zones().reachable().ids().stream()) + .forEach(zone -> { + Cursor object = zones.addObject(); + object.setString("environment", zone.environment().value()); + object.setString("region", zone.region().value()); + object.setString("uri", request.getUri().resolve( + "/configserver/v1/" + zone.environment().value() + "/" + zone.region().value()).toString()); + }); + return new SlimeJsonResponse(slime); + } + + private HttpResponse notFound(Path path) { + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private URI getEndpoint(ZoneId zoneId) { + return CONTROLLER_ZONE.equals(zoneId) ? CONTROLLER_URI : zoneRegistry.getConfigServerVipUri(zoneId); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java new file mode 100644 index 00000000000..9949c2d17bf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author freva + */ +package com.yahoo.vespa.hosted.controller.restapi.configserver; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 5ce679276f7..1a7002c5759 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -1,25 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; import com.yahoo.vespa.hosted.controller.proxy.ProxyException; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.yolean.Exceptions; -import java.io.IOException; +import java.net.URI; +import java.util.List; import java.util.logging.Level; /** @@ -83,8 +84,8 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("No such zone: " + zoneId.value()); } try { - return proxy.handle(new ProxyRequest(request, "/zone/v2/")); - } catch (ProxyException | IOException e) { + return proxy.handle(new ProxyRequest(request, getConfigserverEndpoints(zoneId), path.getRest())); + } catch (ProxyException e) { throw new RuntimeException(e); } } @@ -111,4 +112,13 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { private HttpResponse notFound(Path path) { return ErrorResponse.notFoundError("Nothing at " + path); } + + private List<URI> getConfigserverEndpoints(ZoneId zoneId) { + // TODO: Use config server VIP for all zones that have one + if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { + return List.of(zoneRegistry.getConfigServerVipUri(zoneId)); + } else { + return zoneRegistry.getConfigServerUris(zoneId); + } + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 32bbf3ceb9b..f5158a1ffa2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; @@ -24,8 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,7 +35,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>(); private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); - private List<ZoneApi> zones = new ArrayList<>(); + private List<ZoneApi> zones = List.of(); private SystemName system; private UpgradePolicy upgradePolicy = null; private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); @@ -136,7 +133,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public List<UpgradePolicy> osUpgradePolicies() { - return ImmutableList.copyOf(osUpgradePolicies.values()); + return List.copyOf(osUpgradePolicies.values()); } @Override @@ -176,7 +173,9 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public List<URI> getConfigServerUris(ZoneId zoneId) { - return Collections.singletonList(URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value()))); + return List.of( + URI.create(String.format("https://cfg1.%s.test:4443/", zoneId.value())), + URI.create(String.format("https://cfg2.%s.test:4443/", zoneId.value()))); } @Override @@ -186,11 +185,9 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public List<URI> getConfigServerApiUris(ZoneId zoneId) { - List<URI> uris = new ArrayList<URI>(); - uris.add(URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value()))); - uris.add(URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value()))); - - return uris; + return List.of( + URI.create(String.format("https://cfg.%s.test:4443/", zoneId.value())), + URI.create(String.format("https://cfg.%s.test.vip:4443/", zoneId.value()))); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index 04a987d98d1..d8373cb8928 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -1,20 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; +import com.yahoo.jdisc.http.HttpRequest; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.IOException; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; /** * @author Haakon Dybdahl @@ -26,58 +22,53 @@ public class ProxyRequestTest { @Test public void testEmpty() throws Exception { - exception.expectMessage("Request not set."); - testRequest(null, "/zone/v2/"); + exception.expectMessage("Request must be non-null"); + new ProxyRequest(HttpRequest.Method.GET, null, Map.of(), null, List.of(), "/zone/v2"); } @Test public void testBadUri() throws Exception { - exception.expectMessage("Request not starting with /zone/v2/"); - testRequest(URI.create("http://foo"), "/zone/v2/"); + exception.expectMessage("Request path '/path' does not end with proxy path '/zone/v2/'"); + testRequest("http://domain.tld/path", "/zone/v2/"); } @Test - public void testConfigRequestEmpty() throws Exception { - ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar"), "/zone/v2/"); - assertEquals("foo", proxyRequest.getEnvironment()); - assertEquals("bar", proxyRequest.getRegion()); - assertFalse(proxyRequest.isDiscoveryRequest()); - assertTrue(proxyRequest.getConfigServerRequest().isEmpty()); - - } - - @Test - public void testDiscoveryRequest() throws Exception { - ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo"), "/zone/v2/"); - assertEquals("foo", proxyRequest.getEnvironment()); - assertTrue(proxyRequest.isDiscoveryRequest()); - - } - - @Test - public void testProxyRequest() throws Exception { - ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar/bla/bla/v1/something"), - "/zone/v2/"); - assertEquals("foo", proxyRequest.getEnvironment()); - assertEquals("/bla/bla/v1/something", proxyRequest.getConfigServerRequest()); - } - - @Test - public void testProxyRequestWithParameters() throws Exception { - ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar/something?p=v&q=y"), - "/zone/v2/"); - assertEquals("foo", proxyRequest.getEnvironment()); - assertEquals("/something?p=v&q=y", proxyRequest.getConfigServerRequest()); + public void testUris() throws Exception { + { + // Root request + ProxyRequest request = testRequest("http://controller.domain.tld/my/path", ""); + assertEquals(URI.create("http://controller.domain.tld/my/path/"), request.getControllerPrefixUri()); + assertEquals(URI.create("https://cfg.prod.us-north-1.domain.tld:1234/"), + request.createConfigServerRequestUri(URI.create("https://cfg.prod.us-north-1.domain.tld:1234/"))); + } + + { + // Root request with trailing / + ProxyRequest request = testRequest("http://controller.domain.tld/my/path/", "/"); + assertEquals(URI.create("http://controller.domain.tld/my/path/"), request.getControllerPrefixUri()); + assertEquals(URI.create("https://cfg.prod.us-north-1.domain.tld:1234/"), + request.createConfigServerRequestUri(URI.create("https://cfg.prod.us-north-1.domain.tld:1234/"))); + } + + { + // API path test + ProxyRequest request = testRequest("http://controller.domain.tld:1234/my/path/nodes/v2", "/nodes/v2"); + assertEquals(URI.create("http://controller.domain.tld:1234/my/path/"), request.getControllerPrefixUri()); + assertEquals(URI.create("https://cfg.prod.us-north-1.domain.tld/nodes/v2"), + request.createConfigServerRequestUri(URI.create("https://cfg.prod.us-north-1.domain.tld"))); + } + + { + // API path test with query + ProxyRequest request = testRequest("http://controller.domain.tld:1234/my/path/nodes/v2/?some=thing", "/nodes/v2/"); + assertEquals(URI.create("http://controller.domain.tld:1234/my/path/"), request.getControllerPrefixUri()); + assertEquals(URI.create("https://cfg.prod.us-north-1.domain.tld/nodes/v2/?some=thing"), + request.createConfigServerRequestUri(URI.create("https://cfg.prod.us-north-1.domain.tld"))); + } } - private static ProxyRequest testRequest(URI url, String pathPrefix) throws IOException, ProxyException { - return new ProxyRequest(url, headers("controller:49152"), null, "GET", pathPrefix); + private static ProxyRequest testRequest(String url, String pathPrefix) throws ProxyException { + return new ProxyRequest( + HttpRequest.Method.GET, URI.create(url), Map.of(), null, List.of(), pathPrefix); } - - private static Map<String, List<String>> headers(String hostPort) { - Map<String, List<String>> headers = new HashMap<>(); - headers.put("host", Collections.singletonList(hostPort)); - return Collections.unmodifiableMap(headers); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 8dbd1c4ef61..0aac59321b5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -1,15 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; +import com.yahoo.jdisc.http.HttpRequest; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.net.URI; -import java.util.Collections; -import java.util.HashMap; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -20,50 +19,37 @@ public class ProxyResponseTest { @Test public void testRewriteUrl() throws Exception { - String controllerPrefix = "/zone/v2/"; - URI configServer = URI.create("http://configserver:1234"); - ProxyRequest request = new ProxyRequest(URI.create("http://foo/zone/v2/env/region/configserver"), - headers("controller:49152"), null, "GET", - controllerPrefix); + ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("http://domain.tld/zone/v2/dev/us-north-1/configserver"), + Map.of(), null, List.of(), "configserver"); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", 200, - Optional.of(configServer), + URI.create("http://configserver:1234"), "application/json"); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); proxyResponse.render(outputStream); - String document = new String(outputStream.toByteArray(),"UTF-8"); - assertEquals("response link is http://controller:49152/zone/v2/env/region/bla/bla/", document); + String document = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + assertEquals("response link is http://domain.tld/zone/v2/dev/us-north-1/bla/bla/", document); } @Test public void testRewriteSecureUrl() throws Exception { - String controllerPrefix = "/zone/v2/"; - URI configServer = URI.create("http://configserver:1234"); - ProxyRequest request = new ProxyRequest(URI.create("https://foo/zone/v2/env/region/configserver"), - headers("controller:49152"), null, "GET", - controllerPrefix); + ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("https://domain.tld/zone/v2/prod/eu-south-3/configserver"), + Map.of(), null, List.of(), "configserver"); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", 200, - Optional.of(configServer), + URI.create("http://configserver:1234"), "application/json"); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); proxyResponse.render(outputStream); - String document = new String(outputStream.toByteArray(),"UTF-8"); - assertEquals("response link is https://controller:49152/zone/v2/env/region/bla/bla/", document); + String document = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); + assertEquals("response link is https://domain.tld/zone/v2/prod/eu-south-3/bla/bla/", document); } - - private static Map<String, List<String>> headers(String hostPort) { - Map<String, List<String>> headers = new HashMap<>(); - headers.put("host", Collections.singletonList(hostPort)); - return Collections.unmodifiableMap(headers); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index fb0e92ab7f4..1a53920e8de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -34,12 +34,16 @@ import static org.junit.Assert.assertEquals; */ public class ControllerContainerTest { + private static final AthenzUser hostedOperator = AthenzUser.fromUserId("alice"); private static final AthenzUser defaultUser = AthenzUser.fromUserId("bob"); protected JDisc container; @Before - public void startContainer() { container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable); } + public void startContainer() { + container = JDisc.fromServicesXml(controllerServicesXml(), Networking.disable); + addUserToHostedOperatorRole(hostedOperator); + } @After public void stopContainer() { container.close(); } @@ -92,6 +96,12 @@ public class ControllerContainerTest { " <binding>http://*/zone/v2</binding>\n" + " <binding>http://*/zone/v2/*</binding>\n" + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.configserver.ConfigServerApiHandler'>\n" + + " <binding>http://*/configserver/v1</binding>\n" + + " <binding>http://*/configserver/v1/*</binding>\n" + + " <binding>http://*/api/configserver/v1</binding>\n" + + " <binding>http://*/api/configserver/v1/*</binding>\n" + + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.flags.AuditedFlagsHandler'>\n" + " <binding>http://*/flags/v1</binding>\n" + " <binding>http://*/flags/v1/*</binding>\n" + @@ -147,10 +157,18 @@ public class ControllerContainerTest { return addIdentityToRequest(new Request(uri), defaultUser); } - protected static Request authenticatedRequest(String uri, byte[] body, Request.Method method) { + protected static Request authenticatedRequest(String uri, String body, Request.Method method) { return addIdentityToRequest(new Request(uri, body, method), defaultUser); } + protected static Request operatorRequest(String uri) { + return addIdentityToRequest(new Request(uri), hostedOperator); + } + + protected static Request operatorRequest(String uri, String body, Request.Method method) { + return addIdentityToRequest(new Request(uri, body, method), hostedOperator); + } + protected static Request addIdentityToRequest(Request request, AthenzIdentity identity) { request.getHeaders().put(IDENTITY_HEADER_NAME, identity.getFullName()); return request; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java new file mode 100644 index 00000000000..d08c32a5ea9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java @@ -0,0 +1,144 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.configserver; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.net.URI; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author freva + */ +public class ConfigServerApiHandlerTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/"; + private static final List<ZoneApi> zones = List.of( + ZoneApiMock.fromId("prod.us-north-1"), + ZoneApiMock.fromId("dev.aws-us-north-2"), + ZoneApiMock.fromId("test.us-north-3"), + ZoneApiMock.fromId("staging.us-north-4")); + + private ContainerControllerTester tester; + private ConfigServerProxyMock proxy; + + @Before + public void before() { + ZoneRegistryMock zoneRegistry = (ZoneRegistryMock) container.components() + .getComponent(ZoneRegistryMock.class.getName()); + zoneRegistry.setDefaultRegionForEnvironment(Environment.dev, RegionName.from("us-north-2")) + .setZones(zones); + this.tester = new ContainerControllerTester(container, responseFiles); + this.proxy = (ConfigServerProxyMock) container.components().getComponent(ConfigServerProxyMock.class.getName()); + } + + @Test + public void test_requests() { + // GET /configserver/v1 + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1"), + new File("root.json")); + + // GET /configserver/v1/nodes/v2/node/?recursive=true + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node/?recursive=true"), + "ok"); + assertLastRequest("https://cfg.prod.us-north-1.test.vip:4443/", "GET"); + + // POST /configserver/v1/dev/us-north-2/nodes/v2/command/restart?hostname=node1 + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/dev/aws-us-north-2/nodes/v2/command/restart?hostname=node1", + "", Request.Method.POST), + "ok"); + + // PUT /configserver/v1/prod/us-north-1/nodes/v2/state/dirty/node1 + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/state/dirty/node1", + "", Request.Method.PUT), "ok"); + assertLastRequest("https://cfg.prod.us-north-1.test.vip:4443/", "PUT"); + + // DELETE /configserver/v1/prod/us-north-1/nodes/v2/node/node1 + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/api/configserver/v1/prod/controller/nodes/v2/node/node1", + "", Request.Method.DELETE), "ok"); + assertLastRequest("https://localhost:4443/", "DELETE"); + + // PATCH /configserver/v1/prod/us-north-1/nodes/v2/node/node1 + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/configserver/v1/dev/aws-us-north-2/nodes/v2/node/node1", + "{\"currentRestartGeneration\": 1}", + Request.Method.PATCH), "ok"); + assertLastRequest("https://cfg.dev.aws-us-north-2.test.vip:4443/", "PATCH"); + assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get()); + + assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty()); + } + + @Test + public void test_allowed_apis() { + // GET /configserver/v1/prod/us-north-1 + tester.containerTester().assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1"), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access '/' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}", + 403); + + tester.containerTester().assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-1/application/v2/tenant/vespa"), + "{\"error-code\":\"FORBIDDEN\",\"message\":\"Cannot access '/application/v2/tenant/vespa' through /configserver/v1, following APIs are permitted: /flags/v1/, /nodes/v2/, /orchestrator/v1/\"}", + 403); + } + + @Test + public void test_invalid_requests() { + // POST /configserver/v1/prod/us-north-34/nodes/v2 + tester.containerTester().assertResponse(() -> operatorRequest("http://localhost:8080/configserver/v1/prod/us-north-42/nodes/v2", + "", Request.Method.POST), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such zone: prod.us-north-42\"}", 400); + assertFalse(proxy.lastReceived().isPresent()); + } + + @Test + public void non_operators_are_forbidden() { + // Read request + tester.containerTester().assertResponse(() -> authenticatedRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node"), + "{\n" + + " \"code\" : 403,\n" + + " \"message\" : \"Access denied\"\n" + + "}", 403); + + // Write request + tester.containerTester().assertResponse(() -> authenticatedRequest("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node", "", Request.Method.POST), + "{\n" + + " \"code\" : 403,\n" + + " \"message\" : \"Access denied\"\n" + + "}", 403); + } + + @Test + public void unauthenticated_request_are_unauthorized() { + { + // Read request + Request request = new Request("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node", "", Request.Method.GET); + tester.containerTester().assertResponse(() -> request, "{\n \"message\" : \"Not authenticated\"\n}", 401); + } + + { + // Write request + Request request = new Request("http://localhost:8080/configserver/v1/prod/us-north-1/nodes/v2/node", "", Request.Method.POST); + tester.containerTester().assertResponse(() -> request, "{\n \"message\" : \"Not authenticated\"\n}", 401); + } + } + + + private void assertLastRequest(String target, String method) { + ProxyRequest last = proxy.lastReceived().orElseThrow(); + assertEquals(List.of(URI.create(target)), last.getTargets()); + assertEquals(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(method), last.getMethod()); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/root.json new file mode 100644 index 00000000000..5ccf75d2448 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/responses/root.json @@ -0,0 +1,29 @@ +{ + "zones": [ + { + "environment": "prod", + "region": "controller", + "uri": "http://localhost:8080/configserver/v1/prod/controller" + }, + { + "environment": "prod", + "region": "us-north-1", + "uri": "http://localhost:8080/configserver/v1/prod/us-north-1" + }, + { + "environment": "dev", + "region": "aws-us-north-2", + "uri": "http://localhost:8080/configserver/v1/dev/aws-us-north-2" + }, + { + "environment": "test", + "region": "us-north-3", + "uri": "http://localhost:8080/configserver/v1/test/us-north-3" + }, + { + "environment": "staging", + "region": "us-north-4", + "uri": "http://localhost:8080/configserver/v1/staging/us-north-4" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 74d637499bd..13e82e5132e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.application.container.handler.Request; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -27,48 +25,46 @@ import static org.junit.Assert.assertFalse; public class ControllerApiTest extends ControllerContainerTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/"; - private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator"); private ContainerControllerTester tester; @Before public void before() { - addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR); tester = new ContainerControllerTester(container, responseFiles); } @Test public void testControllerApi() { - tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", new byte[0], Request.Method.GET), new File("root.json")); + tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", "", Request.Method.GET), new File("root.json")); // POST deactivates a maintenance job - tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", "", Request.Method.POST), "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}", 200); // GET a list of all maintenance jobs - tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", new byte[0], Request.Method.GET), + tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", "", Request.Method.GET), new File("maintenance.json")); // DELETE activates maintenance job - tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", "", Request.Method.DELETE), "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}", 200); // DELETE fails to activate unknown maintenance job - tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/foo", + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/foo", "", Request.Method.DELETE), "{\"error-code\":\"NOT_FOUND\",\"message\":\"No job named 'foo'\"}", 404); // DELETE clears inactive flag for maintenance job that has been removed from the code base tester.controller().curator().writeInactiveJobs(Set.of("bar")); - tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar", + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar", "", Request.Method.DELETE), "{\"message\":\"Re-activated job 'bar'\"}", 200); - tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar", + tester.assertResponse(operatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar", "", Request.Method.DELETE), "{\"error-code\":\"NOT_FOUND\",\"message\":\"No job named 'bar'\"}", 404); @@ -79,55 +75,55 @@ public class ControllerApiTest extends ControllerContainerTest { @Test public void testUpgraderApi() { // Get current configuration - tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", new byte[0], Request.Method.GET), + tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", "", Request.Method.GET), "{\"upgradesPerMinute\":100.0,\"confidenceOverrides\":[]}", 200); // Set invalid configuration tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":-1}", Request.Method.PATCH), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":-1}", Request.Method.PATCH), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0, got -1.0\"}", 400); // Ignores unrecognized field tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader","{\"foo\":\"bar\"}", Request.Method.PATCH), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"foo\":\"bar\"}", Request.Method.PATCH), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such modifiable field(s)\"}", 400); // Set upgrades per minute tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":42.0}", Request.Method.PATCH), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":42.0}", Request.Method.PATCH), "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[]}", 200); // Set target major version tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":6}", Request.Method.PATCH), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":6}", Request.Method.PATCH), "{\"upgradesPerMinute\":42.0,\"targetMajorVersion\":6,\"confidenceOverrides\":[]}", 200); // Clear target major version tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":null}", Request.Method.PATCH), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":null}", Request.Method.PATCH), "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[]}", 200); // Override confidence tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "broken", Request.Method.POST), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "broken", Request.Method.POST), "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42\":\"broken\"}]}", 200); // Override confidence for another version tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.43", "broken", Request.Method.POST), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.43", "broken", Request.Method.POST), "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.42\":\"broken\"},{\"6.43\":\"broken\"}]}", 200); // Remove first override tester.assertResponse( - hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "", Request.Method.DELETE), + operatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "", Request.Method.DELETE), "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[{\"6.43\":\"broken\"}]}", 200); @@ -160,8 +156,4 @@ public class ControllerApiTest extends ControllerContainerTest { tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/auditlog/"), new File("auditlog.json")); } - private static Request hostedOperatorRequest(String uri, String body, Request.Method method) { - return addIdentityToRequest(new Request(uri, body, method), HOSTED_VESPA_OPERATOR); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index 19061b61431..40562ba493e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -1,17 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.zone.v2; -import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Request.Method; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; @@ -22,17 +20,17 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author mpolden */ public class ZoneApiTest extends ControllerContainerTest { - private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator"); private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/"; private static final List<ZoneApi> zones = List.of( ZoneApiMock.fromId("prod.us-north-1"), - ZoneApiMock.fromId("dev.us-north-2"), + ZoneApiMock.fromId("dev.aws-us-north-2"), ZoneApiMock.fromId("test.us-north-3"), ZoneApiMock.fromId("staging.us-north-4")); @@ -47,7 +45,6 @@ public class ZoneApiTest extends ControllerContainerTest { .setZones(zones); this.tester = new ContainerControllerTester(container, responseFiles); this.proxy = (ConfigServerProxyMock) container.components().getComponent(ConfigServerProxyMock.class.getName()); - addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR); } @Test @@ -59,53 +56,34 @@ public class ZoneApiTest extends ControllerContainerTest { // GET /zone/v2/prod/us-north-1 tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v2/prod/us-north-1"), "ok"); - assertEquals("prod", proxy.lastReceived().get().getEnvironment()); - assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); - assertEquals("", proxy.lastReceived().get().getConfigServerRequest()); - assertEquals("GET", proxy.lastReceived().get().getMethod()); + + assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "GET"); // GET /zone/v2/nodes/v2/node/?recursive=true tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/?recursive=true"), "ok"); - - assertEquals("prod", proxy.lastReceived().get().getEnvironment()); - assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); - assertEquals("/nodes/v2/node/?recursive=true", proxy.lastReceived().get().getConfigServerRequest()); - assertEquals("GET", proxy.lastReceived().get().getMethod()); + assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "GET"); // POST /zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1 - tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1", - new byte[0], Method.POST), + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/dev/aws-us-north-2/nodes/v2/command/restart?hostname=node1", + "", Method.POST), "ok"); - assertEquals("dev", proxy.lastReceived().get().getEnvironment()); - assertEquals("us-north-2", proxy.lastReceived().get().getRegion()); - assertEquals("/nodes/v2/command/restart?hostname=node1", proxy.lastReceived().get().getConfigServerRequest()); - assertEquals("POST", proxy.lastReceived().get().getMethod()); // PUT /zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1 - tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1", - new byte[0], Method.PUT), "ok"); - assertEquals("prod", proxy.lastReceived().get().getEnvironment()); - assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); - assertEquals("/nodes/v2/state/dirty/node1", proxy.lastReceived().get().getConfigServerRequest()); - assertEquals("PUT", proxy.lastReceived().get().getMethod()); + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1", + "", Method.PUT), "ok"); + assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "PUT"); // DELETE /zone/v2/prod/us-north-1/nodes/v2/node/node1 - tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", - new byte[0], Method.DELETE), "ok"); - assertEquals("prod", proxy.lastReceived().get().getEnvironment()); - assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); - assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest()); - assertEquals("DELETE", proxy.lastReceived().get().getMethod()); + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", + "", Method.DELETE), "ok"); + assertLastRequest(ZoneId.from("prod", "us-north-1"), 2, "DELETE"); // PATCH /zone/v2/prod/us-north-1/nodes/v2/node/node1 - tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/dev/aws-us-north-2/nodes/v2/node/node1", + "{\"currentRestartGeneration\": 1}", Method.PATCH), "ok"); - assertEquals("prod", proxy.lastReceived().get().getEnvironment()); - assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); - assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest()); - assertEquals("PATCH", proxy.lastReceived().get().getMethod()); + assertLastRequest(ZoneId.from("dev", "aws-us-north-2"), 1, "PATCH"); assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get()); assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty()); @@ -114,14 +92,17 @@ public class ZoneApiTest extends ControllerContainerTest { @Test public void test_invalid_requests() { // POST /zone/v2/prod/us-north-34/nodes/v2 - tester.containerTester().assertResponse(hostedOperatorRequest("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2", - new byte[0], Method.POST), + tester.containerTester().assertResponse(operatorRequest("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2", + "", Method.POST), new File("unknown-zone.json"), 400); assertFalse(proxy.lastReceived().isPresent()); } - private static Request hostedOperatorRequest(String uri, byte[] body, Request.Method method) { - return addIdentityToRequest(new Request(uri, body, method), HOSTED_VESPA_OPERATOR); + private void assertLastRequest(ZoneId zoneId, int targets, String method) { + ProxyRequest last = proxy.lastReceived().orElseThrow(); + assertEquals(targets, last.getTargets().size()); + assertTrue(last.getTargets().get(0).toString().contains(zoneId.value())); + assertEquals(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(method), last.getMethod()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json index ab168854267..bd1bc40ba81 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json @@ -1,7 +1,7 @@ { "uris": [ "http://localhost:8080/zone/v2/prod/us-north-1", - "http://localhost:8080/zone/v2/dev/us-north-2", + "http://localhost:8080/zone/v2/dev/aws-us-north-2", "http://localhost:8080/zone/v2/test/us-north-3", "http://localhost:8080/zone/v2/staging/us-north-4" ], @@ -12,7 +12,7 @@ }, { "environment": "dev", - "region": "us-north-2" + "region": "aws-us-north-2" }, { "environment": "test", diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt index a4536f82c21..81e5d86675c 100644 --- a/document/src/vespa/document/select/CMakeLists.txt +++ b/document/src/vespa/document/select/CMakeLists.txt @@ -25,7 +25,6 @@ vespa_add_library(document_select OBJECT gid_filter.cpp invalidconstant.cpp operator.cpp - orderingspecification.cpp result.cpp resultset.cpp resultlist.cpp diff --git a/document/src/vespa/document/select/orderingspecification.cpp b/document/src/vespa/document/select/orderingspecification.cpp deleted file mode 100644 index 60cff313bdb..00000000000 --- a/document/src/vespa/document/select/orderingspecification.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "orderingspecification.h" -#include <vespa/vespalib/stllike/asciistream.h> - -namespace document { - -bool -OrderingSpecification::operator==(const OrderingSpecification& other) const { - return _order == other._order && _orderingStart == other._orderingStart && _widthBits == other._widthBits && _divisionBits == other._divisionBits; -} - -vespalib::string -OrderingSpecification::toString() const { - vespalib::asciistream ost; - ost << (_order == ASCENDING ? "+" : "-") << "," << _widthBits << "," << _divisionBits << "," << _orderingStart; - return ost.str(); -} - -std::ostream& -operator<<(std::ostream& out, const OrderingSpecification& o) -{ - out << o.toString(); - return out; -} - -} diff --git a/document/src/vespa/document/select/orderingspecification.h b/document/src/vespa/document/select/orderingspecification.h deleted file mode 100644 index 370197dd8e9..00000000000 --- a/document/src/vespa/document/select/orderingspecification.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/fastos/types.h> -#include <vespa/vespalib/stllike/string.h> - -namespace document { - -class OrderingSpecification { -public: - typedef std::unique_ptr<OrderingSpecification> UP; - - enum Order { ASCENDING = 0, DESCENDING }; - - OrderingSpecification() - : _order(ASCENDING), _orderingStart(0), _widthBits(0), _divisionBits(0) {}; - - OrderingSpecification(Order order) - : _order(order), _orderingStart(0), _widthBits(0), _divisionBits(0) {}; - - OrderingSpecification(Order order, uint64_t orderingStart, uint16_t widthBits, uint16_t divisionBits) - : _order(order), _orderingStart(orderingStart), _widthBits(widthBits), _divisionBits(divisionBits) {} - - Order getOrder() const { return _order; } - uint64_t getOrderingStart() const { return _orderingStart; } - uint16_t getWidthBits() const { return _widthBits; } - uint16_t getDivisionBits() const { return _divisionBits; } - - bool operator==(const OrderingSpecification& other) const; - - vespalib::string toString() const; - -private: - Order _order; - uint64_t _orderingStart; - uint16_t _widthBits; - uint16_t _divisionBits; -}; - -std::ostream& -operator<<(std::ostream& out, const OrderingSpecification& o); - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index b838fc9d395..b18a4e985f3 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -9,7 +9,6 @@ #include <vespa/vdslib/container/visitorstatistics.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/document/select/orderingspecification.h> #include <vespa/document/fieldvalue/document.h> diff --git a/eval/src/tests/eval/gbdt/.gitignore b/eval/src/tests/eval/gbdt/.gitignore index d0ee762745c..952736e3543 100644 --- a/eval/src/tests/eval/gbdt/.gitignore +++ b/eval/src/tests/eval/gbdt/.gitignore @@ -1 +1,2 @@ /eval_gbdt_benchmark_app +/eval_fast_forest_bench_app diff --git a/eval/src/tests/eval/gbdt/CMakeLists.txt b/eval/src/tests/eval/gbdt/CMakeLists.txt index edbe56e3143..874a2d7bd02 100644 --- a/eval/src/tests/eval/gbdt/CMakeLists.txt +++ b/eval/src/tests/eval/gbdt/CMakeLists.txt @@ -13,3 +13,9 @@ vespa_add_executable(eval_gbdt_benchmark_app vespaeval ) vespa_add_test(NAME eval_gbdt_benchmark_app COMMAND eval_gbdt_benchmark_app BENCHMARK) +vespa_add_executable(eval_fast_forest_bench_app + SOURCES + fast_forest_bench.cpp + DEPENDS + vespaeval +) diff --git a/eval/src/tests/eval/gbdt/fast_forest_bench.cpp b/eval/src/tests/eval/gbdt/fast_forest_bench.cpp new file mode 100644 index 00000000000..76a56bec50c --- /dev/null +++ b/eval/src/tests/eval/gbdt/fast_forest_bench.cpp @@ -0,0 +1,56 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/fast_forest.h> +#include <vespa/eval/eval/vm_forest.h> +#include <vespa/eval/eval/llvm/compiled_function.h> +#include "model.cpp" + +using namespace vespalib::eval; +using namespace vespalib::eval::gbdt; + +template <typename T> +void estimate_cost(size_t num_params, const char *label, const T &impl) { + std::vector<double> inputs_min(num_params, 0.25); + std::vector<double> inputs_med(num_params, 0.50); + std::vector<double> inputs_max(num_params, 0.75); + std::vector<double> inputs_nan(num_params, std::numeric_limits<double>::quiet_NaN()); + double us_min = impl.estimate_cost_us(inputs_min, 5.0); + double us_med = impl.estimate_cost_us(inputs_med, 5.0); + double us_max = impl.estimate_cost_us(inputs_max, 5.0); + double us_nan = impl.estimate_cost_us(inputs_nan, 5.0); + fprintf(stderr, "[%12s] (per 100 eval): [low values] %6.3f ms, [medium values] %6.3f ms, [high values] %6.3f ms, [nan values] %6.3f ms\n", + label, (us_min / 10.0), (us_med / 10.0), (us_max / 10.0), (us_nan / 10.0)); +} + +void run_fast_forest_bench() { + for (size_t tree_size: std::vector<size_t>({8,16,32,64,128,256})) { + for (size_t num_trees: std::vector<size_t>({100, 500, 2500, 5000, 10000})) { + for (size_t max_features: std::vector<size_t>({200})) { + for (size_t less_percent: std::vector<size_t>({100})) { + for (size_t invert_percent: std::vector<size_t>({50})) { + fprintf(stderr, "\n=== features: %zu, num leafs: %zu, num trees: %zu\n", max_features, tree_size, num_trees); + vespalib::string expression = Model().max_features(max_features).less_percent(less_percent).invert_percent(invert_percent).make_forest(num_trees, tree_size); + Function function = Function::parse(expression); + for (size_t min_bits = std::max(size_t(8), tree_size); true; min_bits *= 2) { + auto forest = FastForest::try_convert(function, min_bits, 64); + if (forest) { + estimate_cost(function.num_params(), forest->impl_name().c_str(), *forest); + } + if (min_bits > 64) { + break; + } + } + estimate_cost(function.num_params(), "vm forest", CompiledFunction(function, PassParams::ARRAY, VMForest::optimize_chain)); + } + } + } + } + } + fprintf(stderr, "\n"); +} + +int main(int, char **) { + run_fast_forest_bench(); + return 0; +} diff --git a/eval/src/tests/eval/gbdt/gbdt_test.cpp b/eval/src/tests/eval/gbdt/gbdt_test.cpp index 14fa4510f4d..adb3d22847a 100644 --- a/eval/src/tests/eval/gbdt/gbdt_test.cpp +++ b/eval/src/tests/eval/gbdt/gbdt_test.cpp @@ -17,6 +17,14 @@ using namespace vespalib::eval::gbdt; //----------------------------------------------------------------------------- +bool is_little_endian() { + uint32_t value = 0; + uint8_t bytes[4] = {0, 1, 2, 3}; + static_assert(sizeof(bytes) == sizeof(value)); + memcpy(&value, bytes, sizeof(bytes)); + return (value == 0x03020100); +} + double eval_double(const Function &function, const std::vector<double> ¶ms) { InterpretedFunction ifun(SimpleTensorEngine::ref(), function, NodeTypes()); InterpretedFunction::Context ctx(ifun); @@ -26,6 +34,22 @@ double eval_double(const Function &function, const std::vector<double> ¶ms) double my_resolve(void *ctx, size_t idx) { return ((double*)ctx)[idx]; } +double eval_compiled(const CompiledFunction &cfun, std::vector<double> ¶ms) { + ASSERT_EQUAL(params.size(), cfun.num_params()); + if (cfun.pass_params() == PassParams::ARRAY) { + return cfun.get_function()(¶ms[0]); + } + if (cfun.pass_params() == PassParams::LAZY) { + return cfun.get_lazy_function()(my_resolve, ¶ms[0]); + } + return 31212.0; +} + +double eval_ff(const FastForest &ff, FastForest::Context &ctx, const std::vector<double> ¶ms) { + std::vector<float> my_params(params.begin(), params.end()); + return ff.eval(ctx, &my_params[0]); +} + //----------------------------------------------------------------------------- TEST("require that tree stats can be calculated") { @@ -304,29 +328,18 @@ TEST("require that FastForest model evaluation works") { EXPECT_TRUE(compiled.get_forests().empty()); auto forest = FastForest::try_convert(function); ASSERT_TRUE(forest); - FastForest::Context ctx(*forest); + auto ctx = forest->create_context(); std::vector<double> p1({0.5, 0.5, 0.5}); // all true: 1.0 + 10.0 std::vector<double> p2({2.5, 2.5, 2.5}); // all false: 4.0 + 40.0 std::vector<double> pn(3, std::numeric_limits<double>::quiet_NaN()); // default: 4.0 + 10.0 - EXPECT_EQUAL(forest->eval(ctx, [&p1](size_t i){return p1[i];}), f(&p1[0])); - EXPECT_EQUAL(forest->eval(ctx, [&p2](size_t i){return p2[i];}), f(&p2[0])); - EXPECT_EQUAL(forest->eval(ctx, [&pn](size_t i){return pn[i];}), f(&pn[0])); - EXPECT_EQUAL(forest->eval(ctx, [&p1](size_t i){return p1[i];}), f(&p1[0])); + EXPECT_EQUAL(eval_ff(*forest, *ctx, p1), f(&p1[0])); + EXPECT_EQUAL(eval_ff(*forest, *ctx, p2), f(&p2[0])); + EXPECT_EQUAL(eval_ff(*forest, *ctx, pn), f(&pn[0])); + EXPECT_EQUAL(eval_ff(*forest, *ctx, p1), f(&p1[0])); } //----------------------------------------------------------------------------- -double eval_compiled(const CompiledFunction &cfun, std::vector<double> ¶ms) { - ASSERT_EQUAL(params.size(), cfun.num_params()); - if (cfun.pass_params() == PassParams::ARRAY) { - return cfun.get_function()(¶ms[0]); - } - if (cfun.pass_params() == PassParams::LAZY) { - return cfun.get_lazy_function()(my_resolve, ¶ms[0]); - } - return 31212.0; -} - TEST("require that forests evaluate to approximately the same for all evaluation options") { for (PassParams pass_params: {PassParams::ARRAY, PassParams::LAZY}) { for (size_t tree_size: std::vector<size_t>({20})) { @@ -356,9 +369,36 @@ TEST("require that forests evaluate to approximately the same for all evaluation EXPECT_EQUAL(expected_nan, eval_compiled(deinline, inputs_nan)); EXPECT_EQUAL(expected_nan, eval_compiled(vm_forest, inputs_nan)); if (forest) { - FastForest::Context ctx(*forest); - EXPECT_EQUAL(expected, forest->eval(ctx, [&inputs](size_t i){return inputs[i];})); - EXPECT_EQUAL(expected_nan, forest->eval(ctx, [&inputs_nan](size_t i){return inputs_nan[i];})); + auto ctx = forest->create_context(); + EXPECT_EQUAL(expected, eval_ff(*forest, *ctx, inputs)); + EXPECT_EQUAL(expected_nan, eval_ff(*forest, *ctx, inputs_nan)); + } + } + } + } + } + } +} + +TEST("require that fast forest evaluation is correct for all tree size categories") { + for (size_t tree_size: std::vector<size_t>({7,15,30,61,127})) { + for (size_t num_trees: std::vector<size_t>({127})) { + for (size_t num_features: std::vector<size_t>({35})) { + for (size_t less_percent: std::vector<size_t>({100})) { + for (size_t invert_percent: std::vector<size_t>({50})) { + vespalib::string expression = Model().max_features(num_features).less_percent(less_percent).invert_percent(invert_percent).make_forest(num_trees, tree_size); + Function function = Function::parse(expression); + auto forest = FastForest::try_convert(function); + if ((tree_size <= 64) || is_little_endian()) { + ASSERT_TRUE(forest); + TEST_STATE(forest->impl_name().c_str()); + std::vector<double> inputs(function.num_params(), 0.5); + std::vector<double> inputs_nan(function.num_params(), std::numeric_limits<double>::quiet_NaN()); + double expected = eval_double(function, inputs); + double expected_nan = eval_double(function, inputs_nan); + auto ctx = forest->create_context(); + EXPECT_EQUAL(expected, eval_ff(*forest, *ctx, inputs)); + EXPECT_EQUAL(expected_nan, eval_ff(*forest, *ctx, inputs_nan)); } } } diff --git a/eval/src/tests/eval/gbdt/model.cpp b/eval/src/tests/eval/gbdt/model.cpp index ae1c9bea437..8f0d87a4020 100644 --- a/eval/src/tests/eval/gbdt/model.cpp +++ b/eval/src/tests/eval/gbdt/model.cpp @@ -13,6 +13,7 @@ class Model { private: std::mt19937 _gen; + size_t _max_features; size_t _less_percent; size_t _invert_percent; @@ -32,9 +33,9 @@ private: } std::string make_feature_name() { - size_t max_feature = 2; - while ((max_feature < 1024) && (get_int(0, 99) < 55)) { - max_feature *= 2; + size_t max_feature = 7; + while ((max_feature < _max_features) && (get_int(0, 99) < 55)) { + max_feature = std::min(max_feature * 2, _max_features); } return make_string("feature_%zu", get_int(1, max_feature)); } @@ -60,7 +61,12 @@ private: } public: - explicit Model(size_t seed = 5489u) : _gen(seed), _less_percent(80), _invert_percent(0) {} + explicit Model(size_t seed = 5489u) : _gen(seed), _max_features(1024), _less_percent(80), _invert_percent(0) {} + + Model &max_features(size_t value) { + _max_features = value; + return *this; + } Model &less_percent(size_t value) { _less_percent = value; diff --git a/eval/src/vespa/eval/eval/fast_forest.cpp b/eval/src/vespa/eval/eval/fast_forest.cpp index 927c5355e6a..be58c78d834 100644 --- a/eval/src/vespa/eval/eval/fast_forest.cpp +++ b/eval/src/vespa/eval/eval/fast_forest.cpp @@ -8,16 +8,38 @@ #include <vespa/vespalib/util/benchmark_timer.h> #include <algorithm> #include <cassert> +#include <arpa/inet.h> namespace vespalib::eval::gbdt { namespace { +//----------------------------------------------------------------------------- +// internal concepts used during model creation +//----------------------------------------------------------------------------- + +constexpr size_t bits_per_byte = 8; + +bool is_little_endian() { + uint32_t value = 0; + uint8_t bytes[4] = {0, 1, 2, 3}; + static_assert(sizeof(bytes) == sizeof(value)); + memcpy(&value, bytes, sizeof(bytes)); + return (value == 0x03020100); +} + struct BitRange { uint32_t first; uint32_t last; BitRange(uint32_t bit) : first(bit), last(bit) {} BitRange(uint32_t a, uint32_t b) : first(a), last(b) {} + template <typename T> + size_t covered_words() const { + assert(first <= last); + uint32_t v1 = (first / (bits_per_byte * sizeof(T))); + uint32_t v2 = (last / (bits_per_byte * sizeof(T))); + return ((v2 - v1) + 1); + } static BitRange join(const BitRange &a, const BitRange &b) { assert((a.last + 1) == b.first); return BitRange(a.first, b.last); @@ -43,8 +65,11 @@ struct State { using CmpNodes = std::vector<CmpNode>; std::vector<CmpNodes> cmp_nodes; std::vector<Leafs> leafs; + size_t max_leafs; BitRange encode_node(uint32_t tree_id, const nodes::Node &node); State(size_t num_params, const std::vector<const nodes::Node *> &trees); + size_t num_params() const { return cmp_nodes.size(); } + size_t num_trees() const { return leafs.size(); } ~State() = default; }; @@ -86,12 +111,14 @@ State::encode_node(uint32_t tree_id, const nodes::Node &node) State::State(size_t num_params, const std::vector<const nodes::Node *> &trees) : cmp_nodes(num_params), - leafs(trees.size()) + leafs(trees.size()), + max_leafs(0) { for (uint32_t tree_id = 0; tree_id < trees.size(); ++tree_id) { BitRange leaf_range = encode_node(tree_id, *trees[tree_id]); assert(leaf_range.first == 0); assert((leaf_range.last + 1) == leafs[tree_id].size()); + max_leafs = std::max(max_leafs, leafs[tree_id].size()); } for (CmpNodes &cmp_range: cmp_nodes) { assert(!cmp_range.empty()); @@ -99,134 +126,547 @@ State::State(size_t num_params, const std::vector<const nodes::Node *> &trees) } } +template <typename T> size_t get_lsb(T value) { return vespalib::Optimized::lsbIdx(value); } +template <> size_t get_lsb<uint8_t>(uint8_t value) { return vespalib::Optimized::lsbIdx(uint32_t(value)); } +template <> size_t get_lsb<uint16_t>(uint16_t value) { return vespalib::Optimized::lsbIdx(uint32_t(value)); } + +//----------------------------------------------------------------------------- +// implementation using single value mask per tree +//----------------------------------------------------------------------------- + +template <typename T> vespalib::string fixed_impl_name(); +template <> vespalib::string fixed_impl_name<uint8_t>() { return "ff-fixed<8>"; } +template <> vespalib::string fixed_impl_name<uint16_t>() { return "ff-fixed<16>"; } +template <> vespalib::string fixed_impl_name<uint32_t>() { return "ff-fixed<32>"; } +template <> vespalib::string fixed_impl_name<uint64_t>() { return "ff-fixed<64>"; } + +template <typename T> +constexpr size_t max_leafs() { return (sizeof(T) * bits_per_byte); } + +template <typename T> +struct FixedContext : FastForest::Context { + std::vector<T> masks; + FixedContext(size_t num_trees) : masks(num_trees) {} +}; + +template <typename T> +struct FixedForest : FastForest { + + static T make_mask(const CmpNode &cmp_node) { + BitRange range = cmp_node.false_mask; + size_t num_bits = (sizeof(T) * bits_per_byte); + assert(range.last < num_bits); + assert(range.first <= range.last); + T mask = 0; + for (uint32_t i = 0; i < num_bits; ++i) { + if ((i < range.first) || (i > range.last)) { + mask |= (T(1) << i); + } + } + return mask; + } + + struct Mask { + float value; + uint32_t tree; + T bits; + Mask(float v, uint32_t t, T b) + : value(v), tree(t), bits(b) {} + }; + + struct DMask { + uint32_t tree; + T bits; + DMask(uint32_t t, T b) + : tree(t), bits(b) {} + }; + + std::vector<uint32_t> _mask_sizes; + std::vector<Mask> _masks; + std::vector<uint32_t> _default_offsets; + std::vector<DMask> _default_masks; + std::vector<float> _padded_leafs; + uint32_t _num_trees; + uint32_t _max_leafs; + + FixedForest(const State &state); + static FastForest::UP try_build(const State &state, size_t min_fixed, size_t max_fixed); + + void init_state(T *ctx_masks) const; + static void apply_masks(T *ctx_masks, const Mask *pos, const Mask *end, float limit); + static void apply_masks(T *ctx_masks, const DMask *pos, const DMask *end); + double get_result(const T *ctx_masks) const; + + vespalib::string impl_name() const override { return fixed_impl_name<T>(); } + Context::UP create_context() const override; + double eval(Context &context, const float *params) const override; +}; + +template <typename T> +FixedForest<T>::FixedForest(const State &state) + : _mask_sizes(), + _masks(), + _default_offsets(), + _default_masks(), + _padded_leafs(), + _num_trees(state.num_trees()), + _max_leafs(state.max_leafs) +{ + for (const auto &cmp_nodes: state.cmp_nodes) { + _mask_sizes.emplace_back(cmp_nodes.size()); + _default_offsets.push_back(_default_masks.size()); + for (const CmpNode &cmp_node: cmp_nodes) { + _masks.emplace_back(cmp_node.value, cmp_node.tree_id, make_mask(cmp_node)); + if (cmp_node.false_is_default) { + _default_masks.emplace_back(cmp_node.tree_id, make_mask(cmp_node)); + } + } + } + _default_offsets.push_back(_default_masks.size()); + for (const auto &leafs: state.leafs) { + for (float leaf: leafs) { + _padded_leafs.push_back(leaf); + } + size_t padding = (_max_leafs - leafs.size()); + while (padding-- > 0) { + _padded_leafs.push_back(0.0); + } + } + assert(_padded_leafs.size() == (_num_trees * _max_leafs)); +} + +template <typename T> +FastForest::UP +FixedForest<T>::try_build(const State &state, size_t min_fixed, size_t max_fixed) +{ + if ((max_leafs<T>() >= min_fixed) && + (max_leafs<T>() <= max_fixed) && + (state.max_leafs <= max_leafs<T>())) + { + return std::make_unique<FixedForest<T>>(state); + } + return FastForest::UP(); +} + +template <typename T> +void +FixedForest<T>::init_state(T *ctx_masks) const +{ + memset(ctx_masks, 0xff, _num_trees * sizeof(T)); +} + +template <typename T> +void +FixedForest<T>::apply_masks(T *ctx_masks, const Mask *pos, const Mask *end, float limit) +{ + for (; ((pos+3) < end) && !(limit < pos[3].value); pos += 4) { + ctx_masks[pos[0].tree] &= pos[0].bits; + ctx_masks[pos[1].tree] &= pos[1].bits; + ctx_masks[pos[2].tree] &= pos[2].bits; + ctx_masks[pos[3].tree] &= pos[3].bits; + } + for (; (pos < end) && !(limit < pos->value); ++pos) { + ctx_masks[pos->tree] &= pos->bits; + } +} + +template <typename T> +void +FixedForest<T>::apply_masks(T *ctx_masks, const DMask *pos, const DMask *end) +{ + for (; ((pos+3) < end); pos += 4) { + ctx_masks[pos[0].tree] &= pos[0].bits; + ctx_masks[pos[1].tree] &= pos[1].bits; + ctx_masks[pos[2].tree] &= pos[2].bits; + ctx_masks[pos[3].tree] &= pos[3].bits; + } + for (; (pos < end); ++pos) { + ctx_masks[pos->tree] &= pos->bits; + } +} + +template <typename T> +double +FixedForest<T>::get_result(const T *ctx_masks) const +{ + double result1 = 0.0; + double result2 = 0.0; + const T *ctx_end = (ctx_masks + _num_trees); + const float *leafs = &_padded_leafs[0]; + size_t leaf_cnt = _max_leafs; + for (; (ctx_masks + 3) < ctx_end; ctx_masks += 4, leafs += (leaf_cnt * 4)) { + result1 += leafs[(0 * leaf_cnt) + get_lsb(ctx_masks[0])]; + result2 += leafs[(1 * leaf_cnt) + get_lsb(ctx_masks[1])]; + result1 += leafs[(2 * leaf_cnt) + get_lsb(ctx_masks[2])]; + result2 += leafs[(3 * leaf_cnt) + get_lsb(ctx_masks[3])]; + } + for (; ctx_masks < ctx_end; ++ctx_masks, leafs += leaf_cnt) { + result1 += leafs[get_lsb(*ctx_masks)]; + } + return (result1 + result2); } -struct FastForestBuilder { +template <typename T> +FastForest::Context::UP +FixedForest<T>::create_context() const +{ + return std::make_unique<FixedContext<T>>(_num_trees); +} - static FastForest::MaskType get_mask_type(uint32_t idx1, uint32_t idx2) { - assert(idx1 <= idx2); - if (idx1 == idx2) { - return FastForest::MaskType::ONE; - } else if ((idx1 + 1) == idx2) { - return FastForest::MaskType::TWO; +template <typename T> +double +FixedForest<T>::eval(Context &context, const float *params) const +{ + T *ctx_masks = &static_cast<FixedContext<T>&>(context).masks[0]; + init_state(ctx_masks); + const Mask *mask_pos = &_masks[0]; + const float *param_pos = params; + for (uint32_t size: _mask_sizes) { + float feature = *param_pos++; + if (!std::isnan(feature)) { + apply_masks(ctx_masks, mask_pos, mask_pos + size, feature); } else { - return FastForest::MaskType::MANY; + apply_masks(ctx_masks, + &_default_masks[_default_offsets[(param_pos-params)-1]], + &_default_masks[_default_offsets[(param_pos-params)]]); } + mask_pos += size; } + return get_result(ctx_masks); +} + +//----------------------------------------------------------------------------- +// implementation using multiple words for each tree +//----------------------------------------------------------------------------- + +struct MultiWordContext : FastForest::Context { + std::vector<uint32_t> words; + MultiWordContext(size_t size) : words(size) {} +}; + +struct MultiWordForest : FastForest { + + constexpr static size_t word_size = sizeof(uint32_t); + constexpr static size_t bits_per_word = (word_size * bits_per_byte); + + struct Sizes { + uint32_t fixed; + uint32_t rle; + Sizes(uint32_t f, uint32_t r) : fixed(f), rle(r) {} + }; + + struct Mask { + float value; + uint32_t offset; + union { + uint32_t bits; + uint8_t rle_mask[3]; + }; + Mask(float v, uint32_t word_offset, uint32_t mask_bits) + : value(v), offset(word_offset), bits(mask_bits) {} + Mask(float v, uint32_t byte_offset, uint8_t first_bits, uint8_t empty_bytes, uint8_t last_bits) + : value(v), offset(byte_offset), rle_mask{first_bits, empty_bytes, last_bits} {} + }; - static FastForest::Mask make_mask(const CmpNode &cmp_node) { + struct DMask { + uint32_t offset; + union { + uint32_t bits; + uint8_t rle_mask[3]; + }; + DMask(uint32_t word_offset, uint32_t mask_bits) + : offset(word_offset), bits(mask_bits) {} + DMask(uint32_t byte_offset, uint8_t first_bits, uint8_t empty_bytes, uint8_t last_bits) + : offset(byte_offset), rle_mask{first_bits, empty_bytes, last_bits} {} + }; + + static Mask make_fixed_mask(const CmpNode &cmp_node, size_t words_per_tree) { BitRange range = cmp_node.false_mask; - assert(range.last < (8 * 256)); - assert(range.first <= range.last); - uint32_t idx1 = (range.first / 8); - uint32_t idx2 = (range.last / 8); + assert(range.covered_words<uint32_t>() == 1); + size_t offset = (range.first / bits_per_word); + uint32_t bits = 0; + for (uint32_t i = 0; i < bits_per_word; ++i) { + uint32_t bit = (offset * bits_per_word) + i; + if ((bit < range.first) || (bit > range.last)) { + bits |= (uint32_t(1) << i); + } + } + offset += (words_per_tree * cmp_node.tree_id); + return Mask(cmp_node.value, offset, bits); + } + + static Mask make_rle_mask(const CmpNode &cmp_node, size_t words_per_tree) { + BitRange range = cmp_node.false_mask; + assert(range.covered_words<uint32_t>() > 1); + uint32_t idx1 = (range.first / bits_per_byte); + uint32_t idx2 = (range.last / bits_per_byte); uint8_t bits1 = 0; uint8_t bits2 = 0; - for (uint32_t i = 0; i < 8; ++i) { - uint32_t bit1 = (idx1 * 8) + i; + for (uint32_t i = 0; i < bits_per_byte; ++i) { + uint32_t bit1 = (idx1 * bits_per_byte) + i; if ((bit1 < range.first) || (bit1 > range.last)) { - bits1 |= (1 << i); + bits1 |= (uint8_t(1) << i); } - uint32_t bit2 = (idx2 * 8) + i; + uint32_t bit2 = (idx2 * bits_per_byte) + i; if ((bit2 < range.first) || (bit2 > range.last)) { - bits2 |= (1 << i); + bits2 |= (uint8_t(1) << i); } } - assert(cmp_node.tree_id < (256 * 256)); - return FastForest::Mask(cmp_node.tree_id, get_mask_type(idx1, idx2), cmp_node.false_is_default, - idx1, bits1, idx2, bits2); + uint32_t offset = (idx1 + (word_size * words_per_tree * cmp_node.tree_id)); + uint32_t empty_cnt = ((idx2 - idx1) - 1); + assert(empty_cnt < 256); + return Mask(cmp_node.value, offset, bits1, empty_cnt, bits2); } - static void build(State &state, FastForest &ff) { - for (const auto &cmp_nodes: state.cmp_nodes) { - ff._feature_sizes.push_back(cmp_nodes.size()); - for (const CmpNode &cmp_node: cmp_nodes) { - ff._values.push_back(cmp_node.value); - ff._masks.push_back(make_mask(cmp_node)); + std::vector<Sizes> _mask_sizes; + std::vector<Mask> _masks; + std::vector<Sizes> _default_offsets; + std::vector<DMask> _default_masks; + std::vector<uint32_t> _tree_offsets; + std::vector<float> _leafs; + uint32_t _words_per_tree; + + MultiWordForest(const State &state); + static FastForest::UP try_build(const State &state); + + void init_state(uint32_t *ctx_words) const; + static void apply_fixed_masks(uint32_t *ctx_words, const Mask *pos, const Mask *end, float limit); + static void apply_rle_masks(unsigned char *ctx_bytes, const Mask *pos, const Mask *end, float limit); + static void apply_fixed_masks(uint32_t *ctx_words, const DMask *pos, const DMask *end); + static void apply_rle_masks(unsigned char *ctx_bytes, const DMask *pos, const DMask *end); + static size_t find_leaf(const uint32_t *ctx_words); + double get_result(const uint32_t *ctx_words) const; + + vespalib::string impl_name() const override { return "ff-multiword"; } + Context::UP create_context() const override; + double eval(Context &context, const float *params) const override; +}; + +MultiWordForest::MultiWordForest(const State &state) + : _mask_sizes(), + _masks(), + _default_offsets(), + _default_masks(), + _tree_offsets(), + _leafs(), + _words_per_tree(BitRange(0, state.max_leafs - 1).covered_words<uint32_t>()) +{ + for (const auto &cmp_nodes: state.cmp_nodes) { + std::vector<CmpNode> fixed; + std::vector<CmpNode> rle; + size_t default_fixed_cnt = 0; + for (const CmpNode &cmp_node: cmp_nodes) { + if (cmp_node.false_mask.covered_words<uint32_t>() == 1) { + fixed.push_back(cmp_node); + if (cmp_node.false_is_default) { + ++default_fixed_cnt; + } + } else { + rle.push_back(cmp_node); } } - for (const auto &leafs: state.leafs) { - ff._tree_sizes.push_back(leafs.size()); - for (float leaf: leafs) { - ff._leafs.push_back(leaf); + _mask_sizes.emplace_back(fixed.size(), rle.size()); + _default_offsets.emplace_back(_default_masks.size(), + _default_masks.size() + default_fixed_cnt); + for (const CmpNode &cmp_node: fixed) { + _masks.push_back(make_fixed_mask(cmp_node, _words_per_tree)); + if (cmp_node.false_is_default) { + _default_masks.emplace_back(_masks.back().offset, + _masks.back().bits); + } + } + assert(_default_masks.size() == _default_offsets.back().rle); + for (const CmpNode &cmp_node: rle) { + _masks.push_back(make_rle_mask(cmp_node, _words_per_tree)); + if (cmp_node.false_is_default) { + _default_masks.emplace_back(_masks.back().offset, + _masks.back().rle_mask[0], + _masks.back().rle_mask[1], + _masks.back().rle_mask[2]); } } } -}; + _default_offsets.emplace_back(_default_masks.size(), _default_masks.size()); + for (const auto &leafs: state.leafs) { + _tree_offsets.push_back(_leafs.size()); + for (float leaf: leafs) { + _leafs.push_back(leaf); + } + } +} -FastForest::Context::Context(const FastForest &ff) - : _forest(&ff), - _bytes_per_tree((ff.max_leafs() + 7) / 8), - _bits(_bytes_per_tree * ff.num_trees()) {} +FastForest::UP +MultiWordForest::try_build(const State &state) +{ + if (is_little_endian()) { + if (state.max_leafs <= (bits_per_byte * 256)) { + return std::make_unique<MultiWordForest>(state); + } + } + return FastForest::UP(); +} -FastForest::Context::~Context() = default; +void +MultiWordForest::init_state(uint32_t *ctx_words) const +{ + memset(ctx_words, 0xff, word_size * _words_per_tree * _tree_offsets.size()); +} -FastForest::FastForest() = default; -FastForest::~FastForest() = default; +void +MultiWordForest::apply_fixed_masks(uint32_t *ctx_words, const Mask *pos, const Mask *end, float limit) +{ + for (; ((pos+3) < end) && !(limit < pos[3].value); pos += 4) { + ctx_words[pos[0].offset] &= pos[0].bits; + ctx_words[pos[1].offset] &= pos[1].bits; + ctx_words[pos[2].offset] &= pos[2].bits; + ctx_words[pos[3].offset] &= pos[3].bits; + } + for (; (pos < end) && !(limit < pos->value); ++pos) { + ctx_words[pos->offset] &= pos->bits; + } +} -size_t -FastForest::num_params() const +void +MultiWordForest::apply_rle_masks(unsigned char *ctx_bytes, const Mask *pos, const Mask *end, float limit) { - return _feature_sizes.size(); + for (; (pos < end) && !(limit < pos->value); ++pos) { + unsigned char *dst = (ctx_bytes + pos->offset); + *dst++ &= pos->rle_mask[0]; + for (size_t e = pos->rle_mask[1]; e-- > 0; ) { + *dst++ = 0; + } + *dst &= pos->rle_mask[2]; + } } -size_t -FastForest::num_trees() const +void +MultiWordForest::apply_fixed_masks(uint32_t *ctx_words, const DMask *pos, const DMask *end) { - return _tree_sizes.size(); + for (; ((pos+3) < end); pos += 4) { + ctx_words[pos[0].offset] &= pos[0].bits; + ctx_words[pos[1].offset] &= pos[1].bits; + ctx_words[pos[2].offset] &= pos[2].bits; + ctx_words[pos[3].offset] &= pos[3].bits; + } + for (; (pos < end); ++pos) { + ctx_words[pos->offset] &= pos->bits; + } } -size_t -FastForest::max_leafs() const +void +MultiWordForest::apply_rle_masks(unsigned char *ctx_bytes, const DMask *pos, const DMask *end) { - size_t res = 0; - size_t sum = 0; - for (size_t sz: _tree_sizes) { - res = std::max(res, sz); - sum += sz; + for (; pos < end; ++pos) { + unsigned char *dst = (ctx_bytes + pos->offset); + *dst++ &= pos->rle_mask[0]; + for (size_t e = pos->rle_mask[1]; e-- > 0; ) { + *dst++ = 0; + } + *dst &= pos->rle_mask[2]; } - assert(res <= (8 * 256)); - assert(sum == _leafs.size()); - return res; } -FastForest::UP -FastForest::try_convert(const Function &fun) +size_t +MultiWordForest::find_leaf(const uint32_t *word) { - const auto &root = fun.root(); - if (!root.is_forest()) { - // must be only forest - return FastForest::UP(); + size_t idx = 0; + for (; *word == 0; ++word) { + idx += bits_per_word; } - auto trees = gbdt::extract_trees(root); - if (trees.size() > (256 * 256)) { - // too many trees - return FastForest::UP(); + return (idx + get_lsb(*word)); +} + +double +MultiWordForest::get_result(const uint32_t *ctx_words) const +{ + double result = 0.0; + const float *leafs = &_leafs[0]; + for (size_t tree_offset: _tree_offsets) { + result += leafs[tree_offset + find_leaf(ctx_words)]; + ctx_words += _words_per_tree; } - gbdt::ForestStats stats(trees); - if (stats.total_in_checks > 0) { - // set membership not supported - return FastForest::UP(); + return result; +} + +FastForest::Context::UP +MultiWordForest::create_context() const +{ + return std::make_unique<MultiWordContext>(_words_per_tree * _tree_offsets.size()); +} + +double +MultiWordForest::eval(Context &context, const float *params) const +{ + uint32_t *ctx_words = &static_cast<MultiWordContext&>(context).words[0]; + init_state(ctx_words); + const Mask *mask_pos = &_masks[0]; + const float *param_pos = params; + for (const Sizes &size: _mask_sizes) { + float feature = *param_pos++; + if (!std::isnan(feature)) { + apply_fixed_masks(ctx_words, mask_pos, mask_pos + size.fixed, feature); + apply_rle_masks(reinterpret_cast<unsigned char *>(ctx_words), + mask_pos + size.fixed, mask_pos + size.fixed + size.rle, feature); + } else { + apply_fixed_masks(ctx_words, + &_default_masks[_default_offsets[(param_pos-params)-1].fixed], + &_default_masks[_default_offsets[(param_pos-params)-1].rle]); + apply_rle_masks(reinterpret_cast<unsigned char *>(ctx_words), + &_default_masks[_default_offsets[(param_pos-params)-1].rle], + &_default_masks[_default_offsets[(param_pos-params)].fixed]); + } + mask_pos += (size.fixed + size.rle); } - if (stats.tree_sizes.back().size > (8 * 256)) { - // too many leaf nodes per tree - return FastForest::UP(); + return get_result(ctx_words); +} + +} + +//----------------------------------------------------------------------------- +// outer shell unifying the different implementations +//----------------------------------------------------------------------------- + +FastForest::Context::Context() = default; +FastForest::Context::~Context() = default; + +FastForest::FastForest() = default; +FastForest::~FastForest() = default; + +FastForest::UP +FastForest::try_convert(const Function &fun, size_t min_fixed, size_t max_fixed) +{ + const auto &root = fun.root(); + if (root.is_forest()) { + auto trees = gbdt::extract_trees(root); + gbdt::ForestStats stats(trees); + if (stats.total_in_checks == 0) { + State state(fun.num_params(), trees); + if (auto forest = FixedForest<uint8_t>::try_build(state, min_fixed, max_fixed)) { + return forest; + } + if (auto forest = FixedForest<uint16_t>::try_build(state, min_fixed, max_fixed)) { + return forest; + } + if (auto forest = FixedForest<uint32_t>::try_build(state, min_fixed, max_fixed)) { + return forest; + } + if (auto forest = FixedForest<uint64_t>::try_build(state, min_fixed, max_fixed)) { + return forest; + } + if (auto forest = MultiWordForest::try_build(state)) { + return forest; + } + } } - State state(fun.num_params(), trees); - FastForest::UP res = FastForest::UP(new FastForest()); - FastForestBuilder::build(state, *res); - assert(fun.num_params() == res->num_params()); - assert(trees.size() == res->num_trees()); - return res; + return FastForest::UP(); } double FastForest::estimate_cost_us(const std::vector<double> ¶ms, double budget) const { - Context ctx(*this); - auto get_param = [¶ms](size_t i)->float{ return params[i]; }; - auto self_eval = [&](){ this->eval(ctx, get_param); }; - return BenchmarkTimer::benchmark(self_eval, budget) * 1000.0 * 1000.0; + auto ctx = create_context(); + std::vector<float> my_params(params.begin(), params.end()); + return BenchmarkTimer::benchmark([&](){ eval(*ctx, &my_params[0]); }, budget) * 1000.0 * 1000.0; } } diff --git a/eval/src/vespa/eval/eval/fast_forest.h b/eval/src/vespa/eval/eval/fast_forest.h index df3be2c4aef..27171322454 100644 --- a/eval/src/vespa/eval/eval/fast_forest.h +++ b/eval/src/vespa/eval/eval/fast_forest.h @@ -14,127 +14,28 @@ namespace vespalib::eval::gbdt { * Use modern optimization strategies to improve evaluation * performance of GBDT forests. * - * This model evaluation supports up to 65536 trees with up to 2048 - * leaf nodes in each tree. Comparisons must be on the form 'feature < - * const' or '!(feature >= const)'. The inverted form is used to - * signal that the true branch should be selected when the feature - * values is missing (NaN). + * Comparisons must be on the form 'feature < const' or '!(feature >= + * const)'. The inverted form is used to signal that the true branch + * should be selected when the feature value is missing (NaN). **/ class FastForest { +protected: + FastForest(); public: + virtual ~FastForest(); + using UP = std::unique_ptr<FastForest>; class Context { - friend class FastForest; - private: - const FastForest *_forest; - size_t _bytes_per_tree; - std::vector<uint8_t> _bits; + protected: + Context(); public: - explicit Context(const FastForest &ff); - ~Context(); - }; - -private: - friend struct FastForestBuilder; - - enum class MaskType : uint8_t { ONE, TWO, MANY }; - - struct Mask { - uint16_t tree_id; - MaskType type; - uint8_t false_is_default; - uint8_t first_idx; - uint8_t first_bits; - uint8_t last_idx; - uint8_t last_bits; - Mask(uint16_t tree, MaskType mt, bool def_false, - uint8_t idx1, uint8_t bits1, uint8_t idx2, uint8_t bits2) - : tree_id(tree), type(mt), false_is_default(def_false), - first_idx(idx1), first_bits(bits1), last_idx(idx2), last_bits(bits2) {} + virtual ~Context(); + using UP = std::unique_ptr<Context>; }; - - std::vector<uint32_t> _feature_sizes; - std::vector<float> _values; - std::vector<Mask> _masks; - std::vector<uint32_t> _tree_sizes; - std::vector<float> _leafs; - - FastForest(); - - static void apply_mask(uint8_t *bits, size_t bytes_per_tree, const Mask &mask) { - uint8_t *dst = (bits + (mask.tree_id * bytes_per_tree) + mask.first_idx); - *dst &= mask.first_bits; - if (__builtin_expect(mask.type != MaskType::ONE, false)) { - if (__builtin_expect(mask.type == MaskType::MANY, false)) { - size_t n = (mask.last_idx - mask.first_idx - 1); - while (n-- > 0) { - *++dst = 0x00; - } - } - *++dst &= mask.last_bits; - } - } - - static float find_leaf(const uint8_t *bits, const float *leafs) { - while (__builtin_expect(*bits == 0, true)) { - ++bits; - leafs += 8; - } - return leafs[vespalib::Optimized::lsbIdx(uint32_t(*bits))]; - } - -public: - ~FastForest(); - size_t num_params() const; - size_t num_trees() const; - size_t max_leafs() const; - using UP = std::unique_ptr<FastForest>; - static UP try_convert(const Function &fun); - - template <typename F> - double eval(Context &ctx, F &&f) const { - assert(ctx._forest == this); - size_t bytes_per_tree = ctx._bytes_per_tree; - uint8_t *bits = &ctx._bits[0]; - const float *value_ptr = &_values[0]; - const Mask *mask_ptr = &_masks[0]; - memset(bits, 0xff, ctx._bits.size()); - for (size_t f_idx = 0; f_idx < _feature_sizes.size(); ++f_idx) { - size_t feature_size = _feature_sizes[f_idx]; - float feature_value = f(f_idx); // get param - if (__builtin_expect(std::isnan(feature_value), false)) { - // handle 'missing' input feature - for (size_t i = 0; i < feature_size; ++i) { - const Mask &mask = mask_ptr[i]; - if (mask.false_is_default) { - apply_mask(bits, bytes_per_tree, mask); - } - } - } else { - for (size_t i = 0; i < feature_size; ++i) { - if (__builtin_expect(feature_value < value_ptr[i], false)) { - break; - } else { - apply_mask(bits, bytes_per_tree, mask_ptr[i]); - } - } - } - value_ptr += feature_size; - mask_ptr += feature_size; - } - assert(value_ptr == &*_values.end()); - assert(mask_ptr == &*_masks.end()); - const float *leafs = &_leafs[0]; - double result = 0.0; - for (size_t tree_size: _tree_sizes) { - result += find_leaf(bits, leafs); - bits += bytes_per_tree; - leafs += tree_size; - } - assert(bits == &*ctx._bits.end()); - assert(leafs == &*_leafs.end()); - return result; - } + static UP try_convert(const Function &fun, size_t min_fixed = 8, size_t max_fixed = 64); + virtual vespalib::string impl_name() const = 0; + virtual Context::UP create_context() const = 0; + virtual double eval(Context &context, const float *params) const = 0; double estimate_cost_us(const std::vector<double> ¶ms, double budget = 5.0) const; }; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java index ae1c81195ce..e2167a5cc96 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/BundleCollisionHook.java @@ -8,6 +8,7 @@ import org.osgi.framework.Version; import org.osgi.framework.hooks.bundle.CollisionHook; import org.osgi.framework.hooks.bundle.EventHook; import org.osgi.framework.hooks.bundle.FindHook; +import org.osgi.framework.launch.Framework; import java.util.Collection; import java.util.HashMap; @@ -18,9 +19,10 @@ import java.util.Set; import java.util.logging.Logger; /** - * A bundle {@link CollisionHook} that contains a set of bundles that are allowed to collide with - * bundles that are about to be installed. In order to clean up when bundles are uninstalled, this - * is also a bundle {@link EventHook}. + * A bundle {@link CollisionHook} that contains a set of bundles that are allowed to collide with bundles + * that are about to be installed. This class also implements a {@link FindHook} to provide a consistent + * view of bundles such that the two sets of duplicate bundles are invisible to each other. + * In order to clean up when bundles are uninstalled, this is also a bundle {@link EventHook}. * * Thread safe * @@ -86,9 +88,6 @@ public class BundleCollisionHook implements CollisionHook, EventHook, FindHook { * If the given context represents one of the allowed duplicates, this method filters out all bundles * that are duplicates of the allowed duplicates. Otherwise this method filters out the allowed duplicates, * so they are not visible to other bundles. - * - * NOTE: This hook method is added for a consistent view of the installed bundles, but is not actively - * used by jdisc. The OSGi framework does not use FindHooks when calculating bundle wiring. */ @Override public synchronized void find(BundleContext context, Collection<Bundle> bundles) { @@ -107,7 +106,7 @@ public class BundleCollisionHook implements CollisionHook, EventHook, FindHook { } } } - log.info("Hiding bundles from bundle '" + context.getBundle() + "': " + bundlesToHide); + logHiddenBundles(context, bundlesToHide); bundles.removeAll(bundlesToHide); } @@ -115,6 +114,18 @@ public class BundleCollisionHook implements CollisionHook, EventHook, FindHook { return ! allowedDuplicates.containsKey(bundle) && allowedDuplicates.containsValue(new BsnVersion(bundle)); } + private void logHiddenBundles(BundleContext requestingContext, Set<Bundle> hiddenBundles) { + if (hiddenBundles.isEmpty()) { + log.fine(() -> "No bundles to hide from bundle " + requestingContext.getBundle()); + } else { + if (requestingContext.getBundle() instanceof Framework) { + log.fine(() -> "Requesting bundle is the Framework, so hidden bundles will be visible: " + hiddenBundles); + } else { + log.info("Hiding bundles from bundle '" + requestingContext.getBundle() + "': " + hiddenBundles); + } + } + } + static class BsnVersion { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java index 830e2201966..41acd934333 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java @@ -43,12 +43,16 @@ public class ValuesFetcher { public List<MetricsPacket> fetch(String requestedConsumer) throws JsonRenderingException { ConsumerId consumer = getConsumerOrDefault(requestedConsumer); - return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now()) + return fetchAllMetrics() .stream() .filter(metricsPacket -> metricsPacket.consumers().contains(consumer)) .collect(Collectors.toList()); } + public List<MetricsPacket> fetchAllMetrics() throws JsonRenderingException { + return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now()); + } + private ConsumerId getConsumerOrDefault(String consumer) { if (consumer == null) return DEFAULT_PUBLIC_CONSUMER_ID; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java index cba7fe5c328..4c25796907a 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java @@ -56,7 +56,7 @@ public class YamasHandler extends HttpHandlerBase { private JsonResponse valuesResponse(String consumer) { try { - List<MetricsPacket> metrics = valuesFetcher.fetch(consumer); + List<MetricsPacket> metrics = consumer == null ? valuesFetcher.fetchAllMetrics() : valuesFetcher.fetch(consumer); metrics.addAll(nodeMetricGatherer.gatherMetrics()); // TODO: Currently only add these metrics in this handler. Eventually should be included in all handlers return new JsonResponse(OK, YamasJsonUtil.toYamasArray(metrics, true).serialize()); } catch (JsonRenderingException e) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java index 72c7885d8c1..70ce548916a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeReports.java @@ -3,11 +3,14 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.BaseReport; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; import static com.yahoo.yolean.Exceptions.uncheck; @@ -43,6 +46,25 @@ public class NodeReports { return Optional.ofNullable(reports.get(reportId)).map(r -> uncheck(() -> mapper.treeToValue(r, jacksonClass))); } + /** Gets all reports of the given types and deserialize with the given jacksonClass. */ + public <T> TreeMap<String, T> getReports(Class<T> jacksonClass, BaseReport.Type... types) { + Set<BaseReport.Type> typeSet = Set.of(types); + + return reports.entrySet().stream() + .filter(entry -> { + JsonNode reportType = entry.getValue().findValue(BaseReport.TYPE_FIELD); + if (reportType == null || !reportType.isTextual()) return false; + Optional<BaseReport.Type> type = BaseReport.Type.deserialize(reportType.asText()); + return type.map(typeSet::contains).orElse(false); + }) + .collect(Collectors.toMap( + entry -> entry.getKey(), + entry -> uncheck(() -> mapper.treeToValue(entry.getValue(), jacksonClass)), + (x,y) -> x, // resolves key collisions - cannot happen. + TreeMap::new + )); + } + public void removeReport(String reportId) { if (reports.containsKey(reportId)) { reports.put(reportId, null); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java index eac5f7300ef..1da180226b2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/BaseReport.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.stream.Stream; import static com.yahoo.yolean.Exceptions.uncheck; @@ -23,10 +24,10 @@ import static com.yahoo.yolean.Exceptions.uncheck; * <p><strong>Subclass requirements</strong> * * <ol> - * <li>A subclass mush be a Jackson class that can be mapped to {@link JsonNode} with {@link #toJsonNode()}, + * <li>A subclass must be a Jackson class that can be mapped to {@link JsonNode} with {@link #toJsonNode()}, * and from {@link JsonNode} with {@link #fromJsonNode(JsonNode, Class)}.</li> - * <li>A subclass must override {@link #updates(BaseReport)} and make sure to return false if - * {@code !super.updates(current)}.</li> + * <li>A subclass must override {@link #updates(BaseReport)} and make sure to return true if + * {@code super.updates(current)}.</li> * </ol> * * @author hakonhall @@ -51,10 +52,18 @@ public class BaseReport { public enum Type { /** The default type if none given, or not recognized. */ UNSPECIFIED, + /** A program to be executed once. */ + ONCE, /** The host has a soft failure and should be parked for manual inspection. */ SOFT_FAIL, /** The host has a hard failure and should be given back to siteops. */ - HARD_FAIL + HARD_FAIL; + + public static Optional<Type> deserialize(String typeString) { + return Stream.of(Type.values()).filter(type -> type.name().equalsIgnoreCase(typeString)).findAny(); + } + + public String serialize() { return name(); } } @JsonCreator diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index cc9c8c87272..2017f0b3e6d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -64,10 +64,7 @@ public class DockerOperationsImpl implements DockerOperations { public void createContainer(NodeAgentContext context, ContainerData containerData, ContainerResources containerResources) { context.log(logger, "Creating container"); - // IPv6 - Assume always valid - Inet6Address ipV6Address = ipAddresses.getIPv6Address(context.node().hostname()).orElseThrow( - () -> new RuntimeException("Unable to find a valid IPv6 address for " + context.node().hostname() + - ". Missing an AAAA DNS entry?")); + Optional<Inet6Address> ipV6Address = ipAddresses.getIPv6Address(context.node().hostname()); Docker.CreateContainerCommand command = docker.createContainerCommand( context.node().wantedDockerImage().get(), context.containerName()) @@ -97,14 +94,19 @@ public class DockerOperationsImpl implements DockerOperations { command.withNetworkMode(networking.getDockerNetworkMode()); if (networking == DockerNetworking.NPT) { - InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, IPV6_NPT_PREFIX, 8); - command.withIpAddress(ipV6Local); + Optional<InetAddress> ipV6Local = ipV6Address.map(ip -> IPAddresses.prefixTranslate(ip, IPV6_NPT_PREFIX, 8)); + ipV6Local.ifPresent(command::withIpAddress); // IPv4 - Only present for some containers Optional<InetAddress> ipV4Local = ipAddresses.getIPv4Address(context.node().hostname()) .map(ipV4Address -> IPAddresses.prefixTranslate(ipV4Address, IPV4_NPT_PREFIX, 2)); ipV4Local.ifPresent(command::withIpAddress); + if (ipV4Local.isEmpty() && ipV6Address.isEmpty()) { + throw new IllegalArgumentException("Container " + context.node().hostname() + " with " + + networking + " networking must have at least 1 IP address, " + + "but found none"); + } addEtcHosts(containerData, context.node().hostname(), ipV4Local, ipV6Local); } @@ -117,7 +119,7 @@ public class DockerOperationsImpl implements DockerOperations { void addEtcHosts(ContainerData containerData, String hostname, Optional<InetAddress> ipV4Local, - InetAddress ipV6Local) { + Optional<InetAddress> ipV6Local) { // The default /etc/hosts in a Docker container contains one entry for the host, // mapping the hostname to the Docker-assigned IPv4 address. // @@ -137,8 +139,8 @@ public class DockerOperationsImpl implements DockerOperations { "fe00::0\tip6-localnet\n" + "ff00::0\tip6-mcastprefix\n" + "ff02::1\tip6-allnodes\n" + - "ff02::2\tip6-allrouters\n" + - ipV6Local.getHostAddress() + '\t' + hostname + '\n'); + "ff02::2\tip6-allrouters\n"); + ipV6Local.ifPresent(ipv6 -> etcHosts.append(ipv6.getHostAddress()).append('\t').append(hostname).append('\n')); ipV4Local.ifPresent(ipv4 -> etcHosts.append(ipv4.getHostAddress()).append('\t').append(hostname).append('\n')); containerData.addFile(Paths.get("/etc/hosts"), etcHosts.toString()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java index cbc8ffbf1b7..bc88702b0fc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; +import com.yahoo.vespa.hosted.node.admin.task.util.text.SnippetGenerator; + /** * Base class for child process related exceptions, with a util to build an error message * that includes a large part of the output. @@ -10,10 +12,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; */ @SuppressWarnings("serial") public abstract class ChildProcessException extends RuntimeException { - private static final int MAX_OUTPUT_PREFIX = 200; - private static final int MAX_OUTPUT_SUFFIX = 200; - // Omitting a number of chars less than 10 or less than 10% would be ridiculous. - private static final int MAX_OUTPUT_SLACK = Math.max(10, (10 * (MAX_OUTPUT_PREFIX + MAX_OUTPUT_SUFFIX)) / 100); + private static final SnippetGenerator snippetGenerator = new SnippetGenerator(); /** * An exception with a message of the following format: @@ -36,44 +35,13 @@ public abstract class ChildProcessException extends RuntimeException { super(makeSnippet(problem, commandLine, possiblyHugeOutput), cause); } - private static String makeSnippet(String problem, - String commandLine, - String possiblyHugeOutput) { - return makeSnippet( - problem, - commandLine, - possiblyHugeOutput, - MAX_OUTPUT_PREFIX, - MAX_OUTPUT_SUFFIX, - MAX_OUTPUT_SLACK); - } - - // Package-private instead of private for testing. - static String makeSnippet(String problem, - String commandLine, - String possiblyHugeOutput, - int maxOutputPrefix, - int maxOutputSuffix, - int maxOutputSlack) { - StringBuilder stringBuilder = new StringBuilder() - .append("Command '") - .append(commandLine) - .append("' ") - .append(problem) - .append(": stdout/stderr: '"); - - if (possiblyHugeOutput.length() <= maxOutputPrefix + maxOutputSuffix + maxOutputSlack) { - stringBuilder.append(possiblyHugeOutput); - } else { - stringBuilder.append(possiblyHugeOutput, 0, maxOutputPrefix) - .append("... [") - .append(possiblyHugeOutput.length() - maxOutputPrefix - maxOutputSuffix) - .append(" chars omitted] ...") - .append(possiblyHugeOutput.substring(possiblyHugeOutput.length() - maxOutputSuffix)); - } - - stringBuilder.append("'"); - - return stringBuilder.toString(); + private static String makeSnippet(String problem, String commandLine, String possiblyHugeOutput) { + return "Command '" + + commandLine + + "' " + + problem + + ": stdout/stderr: '" + + snippetGenerator.makeSnippet(possiblyHugeOutput, 500) + + "'"; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGenerator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGenerator.java new file mode 100644 index 00000000000..7694260d1f7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGenerator.java @@ -0,0 +1,32 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +/** + * @author hakon + */ +public class SnippetGenerator { + + private static final String OMIT_PREFIX = "[..."; + private static final String OMIT_SUFFIX = " chars omitted]"; + private static final int ASSUMED_OMIT_TEXT_LENGTH = OMIT_PREFIX.length() + 4 + OMIT_SUFFIX.length(); + + /** Returns a snippet of approximate size. */ + public String makeSnippet(String text, int sizeHint) { + if (text.length() <= Math.max(sizeHint, ASSUMED_OMIT_TEXT_LENGTH)) return text; + + int maxSuffixLength = Math.max(0, (sizeHint - ASSUMED_OMIT_TEXT_LENGTH) / 2); + int maxPrefixLength = Math.max(0, sizeHint - ASSUMED_OMIT_TEXT_LENGTH - maxSuffixLength); + String sizeString = Integer.toString(text.length() - maxPrefixLength - maxSuffixLength); + + // It would be silly to return a snippet when the full text is barely longer. + // Note: Say ASSUMED_OMIT_TEXT_LENGTH=23: text will be returned whenever sizeHint<23 and text.length()<28. + int snippetLength = maxPrefixLength + OMIT_PREFIX.length() + sizeString.length() + OMIT_SUFFIX.length() + maxSuffixLength; + if (text.length() <= 1.05 * snippetLength + 5) return text; + + return text.substring(0, maxPrefixLength) + + OMIT_PREFIX + + sizeString + + OMIT_SUFFIX + + text.substring(text.length() - maxSuffixLength); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index 5091e59e175..48a9e8ca039 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -98,7 +98,7 @@ public class DockerOperationsImplTest { InetAddress ipV6Local = InetAddresses.forString("::1"); InetAddress ipV4Local = InetAddresses.forString("127.0.0.1"); - dockerOperations.addEtcHosts(containerData, hostname, Optional.empty(), ipV6Local); + dockerOperations.addEtcHosts(containerData, hostname, Optional.empty(), Optional.of(ipV6Local)); verify(containerData, times(1)).addFile( Paths.get("/etc/hosts"), @@ -111,7 +111,7 @@ public class DockerOperationsImplTest { "ff02::2 ip6-allrouters\n" + "0:0:0:0:0:0:0:1 hostname\n"); - dockerOperations.addEtcHosts(containerData, hostname, Optional.of(ipV4Local), ipV6Local); + dockerOperations.addEtcHosts(containerData, hostname, Optional.of(ipV4Local), Optional.of(ipV6Local)); verify(containerData, times(1)).addFile( Paths.get("/etc/hosts"), diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGeneratorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGeneratorTest.java new file mode 100644 index 00000000000..8a67052a286 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/text/SnippetGeneratorTest.java @@ -0,0 +1,59 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author hakon + */ +public class SnippetGeneratorTest { + private final SnippetGenerator generator = new SnippetGenerator(); + + private void assertSnippet(String text, int sizeHint, String expectedSnippet) { + assertEquals(expectedSnippet, generator.makeSnippet(text, sizeHint)); + } + + @Test + public void prefixSnippetForReallySmallSizeHint() { + assertSnippet( + "This is a long text that should be snippeted", 0, + "[...44 chars omitted]"); + + assertSnippet( + "This is a long text that should be snippeted", 1, + "[...44 chars omitted]"); + } + + @Test + public void snippet() { + assertSnippet( + "This is a long text that should be snippeted", 23, + "[...44 chars omitted]"); + + assertSnippet( + "This is a long text that should be snippeted", 24, + "T[...43 chars omitted]"); + + assertSnippet( + "This is a long text that should be snippeted", 30, + "This[...37 chars omitted]ted"); + + } + + @Test + public void noShorteningNeeded() { + assertSnippet( + "This is a long text that should be snippeted", 39, + "This is [...28 chars omitted]nippeted"); + + assertSnippet( + "This is a long text that should be snippeted", 40, + "This is a long text that should be snippeted"); + + assertSnippet( + "This is a long text that should be snippeted", 50, + "This is a long text that should be snippeted"); + } +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 112915b63cd..7c5e529fb03 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -166,7 +166,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { failedExpirerInterval = Duration.ofMinutes(10); provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); - capacityReportInterval = Duration.ofHours(1); + capacityReportInterval = Duration.ofMinutes(10); metricsInterval = Duration.ofMinutes(1); infrastructureProvisionInterval = Duration.ofMinutes(1); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java index 832e0094121..5d31e262d2a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi.v2; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -69,4 +70,13 @@ public class NodeSerializer { } } + public String toString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index d12150408c4..64cc2691010 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -239,7 +239,6 @@ public class NodesApiHandler extends LoggingRequestHandler { private Flavor flavorFromSlime(Inspector inspector) { Inspector flavorInspector = inspector.field("flavor"); - log.info("flavorFromSlime: " + flavorInspector.valid()); if (!flavorInspector.valid()) { return new Flavor(new NodeResources( requiredField(inspector, "minCpuCores", Inspector::asDouble), @@ -260,7 +259,6 @@ public class NodesApiHandler extends LoggingRequestHandler { flavor = flavor.with(flavor.resources().withBandwidthGbps(inspector.field("bandwidthGbps").asDouble())); if (inspector.field("fastDisk").valid()) flavor = flavor.with(flavor.resources().withDiskSpeed(inspector.field("fastDisk").asBool() ? fast : slow)); - log.info("should not be here"); return flavor; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java index 4ebd458aa15..018d14ef6e0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; @@ -162,6 +163,7 @@ class NodesResponse extends HttpResponse { object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); object.setString("wantedDockerImage", dockerImageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString()); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); + toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); orchestrator.apply(new HostName(node.hostname())) .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN) @@ -212,6 +214,14 @@ class NodesResponse extends HttpResponse { } } + private void toSlime(NodeResources resources, Cursor object) { + object.setDouble("vcpu", resources.vcpu()); + object.setDouble("memoryGb", resources.memoryGb()); + object.setDouble("diskGb", resources.diskGb()); + object.setDouble("bandwidthGbps", resources.bandwidthGbps()); + object.setString("diskSpeed", serializer.toString(resources.diskSpeed())); + } + // Hack: For non-docker noder, return current docker image as default prefix + current Vespa version // TODO: Remove current + wanted docker image from response for non-docker types private Optional<DockerImage> currentDockerImage(Node node) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json index 16dcd9a3fbd..e419d709490 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json @@ -30,6 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json index 31bdfe7d6e3..bb4ebd3588c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json index f5068924084..7a1c873040a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json index 8aeb3f844ef..7a15e49c4c1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json index a7e9292396c..fb9d8675431 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json index 60825d0925f..3e750c75403 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json index b669bad3704..58aa6c4f60e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json index ba6f02efb21..80b09c7460c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json index 24e64248b1c..a892b6ee142 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":4.0, "memoryGb":32.0, "diskGb":1600.0, "bandwidthGbps":20.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index a291c10e540..e8a6f623b5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index 1f60631296d..82a7d26d6c6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -30,6 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json index 7cf66c97603..2ffccb2cc92 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json index 4235ad5ad03..c406a2a9de0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 0bd3df05449..8fbd23e4d84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index fe0484e31ba..9d2cec94367 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -30,6 +30,7 @@ "currentRestartGeneration": 1, "wantedDockerImage": "docker.domain.tld/my/image:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 3, "currentRebootGeneration": 1, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index ca5b3218bc7..30d0e292b3e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -30,6 +30,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json index 788efb86d6b..01e2fcf7e09 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json index 26de9d1efd6..39cce29e38e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json index 03db738cd2e..01fdb4d9e72 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index 6fa1ccdb6fc..5a74e65c43a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -29,6 +29,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast" }, "allowedToBeDown": false, "rebootGeneration": 1, "currentRebootGeneration": 0, diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index c72be5b23e9..d2170cf2d1d 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1298,7 +1298,7 @@ ], "methods": [ "public void <init>()", - "public final int hashCode()", + "public int hashCode()", "public final boolean equals(java.lang.Object)", "public final java.lang.String toString()", "public abstract java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", @@ -1519,6 +1519,7 @@ "public void <init>(java.lang.String, java.util.List, java.lang.String)", "public void <init>(com.yahoo.searchlib.rankingexpression.Reference)", "public java.lang.String getName()", + "public int hashCode()", "public com.yahoo.searchlib.rankingexpression.rule.Arguments getArguments()", "public com.yahoo.searchlib.rankingexpression.rule.ReferenceNode setArguments(java.util.List)", "public java.lang.String getOutput()", diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java index dba0da7301d..dfcdf1e2662 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java @@ -19,7 +19,7 @@ import java.util.Deque; public abstract class ExpressionNode implements Serializable { @Override - public final int hashCode() { + public int hashCode() { return toString().hashCode(); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index e15ce158e83..7312863fa26 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -42,6 +42,9 @@ public final class ReferenceNode extends CompositeNode { return reference.name(); } + public int hashCode() { + return reference.hashCode(); + } /** Returns the arguments, never null */ public Arguments getArguments() { return reference.arguments(); } diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index d0a04e2a007..5d2dad39ed0 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -85,7 +85,7 @@ public: constexpr uint32_t DOCID_LIMIT = 3; -bool search(const Node &node, IAttributeManager &attribute_manager) { +bool search(const Node &node, IAttributeManager &attribute_manager, bool expect_attribute_search_context = true) { AttributeContext ac(attribute_manager); FakeRequestContext requestContext(&ac); MatchData::UP md(MatchData::makeTestInstance(1, 1)); @@ -94,6 +94,11 @@ bool search(const Node &node, IAttributeManager &attribute_manager) { ASSERT_TRUE(result.get()); EXPECT_TRUE(!result->getState().estimate().empty); EXPECT_EQUAL(3u, result->getState().estimate().estHits); + if (expect_attribute_search_context) { + EXPECT_TRUE(result->get_attribute_search_context() != nullptr); + } else { + EXPECT_TRUE(result->get_attribute_search_context() == nullptr); + } result->fetchPostings(true); result->setDocIdLimit(DOCID_LIMIT); SearchIterator::UP iterator = result->createSearch(*md, true); @@ -181,13 +186,13 @@ TEST("requireThatLocationTermsWork") { MyAttributeManager attribute_manager = makeAttributeManager(int64_t(0xcc)); SimpleLocationTerm node(Location(Point(10, 10), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(search(node, attribute_manager)); + EXPECT_TRUE(search(node, attribute_manager, false)); node = SimpleLocationTerm(Location(Point(100, 100), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(!search(node, attribute_manager)); + EXPECT_TRUE(!search(node, attribute_manager, false)); node = SimpleLocationTerm(Location(Point(13, 13), 4, 0), field, 0, Weight(0)); - EXPECT_TRUE(!search(node, attribute_manager)); + EXPECT_TRUE(!search(node, attribute_manager, false)); node = SimpleLocationTerm(Location(Point(10, 13), 3, 0), field, 0, Weight(0)); - EXPECT_TRUE(search(node, attribute_manager)); + EXPECT_TRUE(search(node, attribute_manager, false)); } TEST("requireThatFastSearchLocationTermsWork") { diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp index 8329398f8af..f7d49d6a06a 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp @@ -266,11 +266,6 @@ private: void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, const Config &cfg, int64_t maxValue); void requireThatOutOfBoundsSearchTermGivesZeroHits(); - - template <typename AttributeType, typename ValueType> - void requireThatSearchIteratorExposesSearchContext(const ConfigMap &cfg, ValueType value, const vespalib::string &searchTerm); - void requireThatSearchIteratorExposesSearchContext(); - // init maps with config objects void initIntegerConfig(); void initFloatConfig(); @@ -1830,47 +1825,6 @@ SearchContextTest::requireThatOutOfBoundsSearchTermGivesZeroHits() } void -assertSearchIteratorExposesSearchContext(search::attribute::ISearchContext &ctx) -{ - ASSERT_TRUE(ctx.valid()); - ctx.fetchPostings(true); - TermFieldMatchData dummy; - SearchBasePtr itr = ctx.createIterator(&dummy, true); - EXPECT_TRUE(itr->getAttributeSearchContext() != nullptr); - EXPECT_EQUAL(&ctx, itr->getAttributeSearchContext()); -} - -template <typename AttributeType, typename ValueType> -void -SearchContextTest::requireThatSearchIteratorExposesSearchContext(const ConfigMap &cfgMap, - ValueType value, - const vespalib::string &searchTerm) -{ - vespalib::string attrSuffix = "-itr-exposes-ctx"; - std::vector<ValueType> values = {value}; - for (const auto &cfg : cfgMap) { - vespalib::string attrName = cfg.first + attrSuffix; - AttributePtr attr = AttributeFactory::createAttribute(attrName, cfg.second); - addDocs(*attr, 2); - auto &concreteAttr = dynamic_cast<AttributeType &>(*attr); - if (attr->hasMultiValue()) { - fillAttribute(concreteAttr, values); - } else { - resetAttribute(concreteAttr, value); - } - assertSearchIteratorExposesSearchContext(*getSearch(*attr, searchTerm)); - } -} - -void -SearchContextTest::requireThatSearchIteratorExposesSearchContext() -{ - requireThatSearchIteratorExposesSearchContext<IntegerAttribute, largeint_t>(_integerCfg, 3, "3"); - requireThatSearchIteratorExposesSearchContext<FloatingPointAttribute, double>(_floatCfg, 5.7, "5.7"); - requireThatSearchIteratorExposesSearchContext<StringAttribute, vespalib::string>(_stringCfg, "foo", "foo"); -} - -void SearchContextTest::initIntegerConfig() { { // CollectionType::SINGLE @@ -2000,7 +1954,6 @@ SearchContextTest::Main() TEST_DO(requireThatInvalidSearchTermGivesZeroHits()); TEST_DO(requireThatFlagAttributeHandlesTheByteRange()); TEST_DO(requireThatOutOfBoundsSearchTermGivesZeroHits()); - TEST_DO(requireThatSearchIteratorExposesSearchContext()); TEST_DONE(); } diff --git a/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp b/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp index 6cef4479439..6fc75c8e696 100644 --- a/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp +++ b/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp @@ -371,7 +371,7 @@ TEST_F(FakeSearchableTest, require_that_relevant_data_can_be_obtained_from_fake_ MatchData::UP md = MatchData::makeTestInstance(100, 10); bp->fetchPostings(false); SearchIterator::UP search = bp->createSearch(*md, false); - EXPECT_EQ(bp->get_attribute_search_context(), search->getAttributeSearchContext()); + EXPECT_TRUE(bp->get_attribute_search_context() != nullptr); const auto *attr_ctx = bp->get_attribute_search_context(); ASSERT_TRUE(attr_ctx); EXPECT_EQ(attr_ctx->attributeName(), "attrfoo"); diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index 4d81f46a65c..6d304b61663 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -40,7 +40,6 @@ public: _matchPosition(_matchData->populate_fixed()) { } Trinary is_strict() const override { return Trinary::False; } - const attribute::ISearchContext *getAttributeSearchContext() const override { return &_baseSearchCtx; } }; diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index a4b2280fa57..0246f96f1df 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -50,10 +50,11 @@ class FastForestExecutor : public fef::FeatureExecutor { private: const FastForest &_forest; - FastForest::Context _ctx; + FastForest::Context::UP _ctx; + ArrayRef<float> _params; public: - FastForestExecutor(const FastForest &forest); + FastForestExecutor(ArrayRef<float> param_space, const FastForest &forest); bool isPure() override { return true; } void execute(uint32_t docId) override; }; @@ -128,18 +129,27 @@ public: //----------------------------------------------------------------------------- -FastForestExecutor::FastForestExecutor(const FastForest &forest) +FastForestExecutor::FastForestExecutor(ArrayRef<float> param_space, const FastForest &forest) : _forest(forest), - _ctx(_forest) + _ctx(_forest.create_context()), + _params(param_space) { } void FastForestExecutor::execute(uint32_t) { - const auto ¶ms = inputs(); - double result = _forest.eval(_ctx, [¶ms](size_t p){ return params.get_number(p); }); - outputs().set_number(0, result); + size_t i = 0; + for (; (i + 3) < _params.size(); i += 4) { + _params[i+0] = inputs().get_number(i+0); + _params[i+1] = inputs().get_number(i+1); + _params[i+2] = inputs().get_number(i+2); + _params[i+3] = inputs().get_number(i+3); + } + for (; i < _params.size(); ++i) { + _params[i] = inputs().get_number(i); + } + outputs().set_number(0, _forest.eval(*_ctx, &_params[0])); } //----------------------------------------------------------------------------- @@ -342,7 +352,8 @@ RankingExpressionBlueprint::createExecutor(const fef::IQueryEnvironment &env, ve return stash.create<InterpretedRankingExpressionExecutor>(*_interpreted_function, input_is_object); } if (_fast_forest) { - return stash.create<FastForestExecutor>(*_fast_forest); + ArrayRef<float> param_space = stash.create_array<float>(_input_is_object.size(), 0.0); + return stash.create<FastForestExecutor>(param_space, *_fast_forest); } assert(_compile_token.get() != nullptr); // will be nullptr for VERIFY_SETUP feature motivation if (_compile_token->get().pass_params() == PassParams::ARRAY) { diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.h b/searchlib/src/vespa/searchlib/queryeval/fake_search.h index e629e849408..f5b95a94e99 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.h @@ -43,7 +43,6 @@ public: void doUnpack(uint32_t docid) override; const PostingInfo *getPostingInfo() const override { return _result.postingInfo(); } void visitMembers(vespalib::ObjectVisitor &visitor) const override; - const attribute::ISearchContext *getAttributeSearchContext() const override { return _ctx; } }; } // namespace queryeval diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 9b84136e67c..8ca8ef0f102 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -77,7 +77,7 @@ SameElementBlueprint::create_same_element_search(bool strict) const for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); SearchIterator::UP child = _terms[i]->createSearch(*md, (strict && (i == 0))); - const attribute::ISearchContext *context = child->getAttributeSearchContext(); + const attribute::ISearchContext *context = _terms[i]->get_attribute_search_context(); if (context == nullptr) { children[i] = std::move(child); childMatch.add(childState.field(0).resolve(*md)); diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp index 9450aceb2be..38830262714 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp @@ -117,12 +117,6 @@ SearchIterator::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "endid", _endid); } -const attribute::ISearchContext * -SearchIterator::getAttributeSearchContext() const -{ - return nullptr; -} - } //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h index 27494e08f90..8ea45646af8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h @@ -356,8 +356,6 @@ public: virtual Trinary is_strict() const { return Trinary::Undefined; } - /** return the underlying attribute search context (or null if none available) */ - virtual const attribute::ISearchContext *getAttributeSearchContext() const; }; } diff --git a/storage/src/vespa/storage/distributor/operations/external/visitororder.h b/storage/src/vespa/storage/distributor/operations/external/visitororder.h index 89a5d9e3734..f33500bfc1f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitororder.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitororder.h @@ -1,34 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <climits> +#include <vespa/document/bucket/bucketid.h> namespace storage::distributor { struct VisitorOrder { - document::OrderingSpecification _ordering; - VisitorOrder() - : _ordering() {} - - document::BucketId::Type getOrder(const document::BucketId& bid) { - int32_t orderBitCount = _ordering.getWidthBits() - - _ordering.getDivisionBits(); - document::BucketId::Type order = bid.withoutCountBits(); - order >>= 32; - order <<= 64 - orderBitCount; - order = document::BucketId::reverse(order); - return order; - } - - document::BucketId::Type padOnesRight(const document::BucketId::Type& id, - int32_t count) { - document::BucketId::Type res = id; - document::BucketId::Type one = 1; - for (int32_t i=0; i<count; i++) { - res |= (one << i); - } - return res; - } + VisitorOrder() { } bool operator()(const document::BucketId& a, const document::BucketId& b) { if (a == document::BucketId(INT_MAX) || @@ -39,40 +18,7 @@ struct VisitorOrder { b == document::BucketId(INT_MAX)) { return true; // All after null, non after max } - int32_t orderBitCount = _ordering.getWidthBits() - - _ordering.getDivisionBits(); - int32_t aOrderBitsUsed = std::max((int32_t)a.getUsedBits() - 32, 0); - int32_t bOrderBitsUsed = std::max((int32_t)b.getUsedBits() - 32, 0); - if (orderBitCount <= 0 || - aOrderBitsUsed == 0 || - bOrderBitsUsed == 0) { - return (a.toKey() < b.toKey()); // Reversed bucket id order - } - - document::BucketId::Type aOrder = getOrder(a); - document::BucketId::Type bOrder = getOrder(b); - - document::BucketId::Type sOrder = _ordering.getOrderingStart(); - sOrder <<= 64 - _ordering.getWidthBits(); - sOrder >>= 64 - orderBitCount; - - if (_ordering.getOrder() == document::OrderingSpecification::ASCENDING) { - aOrder = padOnesRight(aOrder, orderBitCount - aOrderBitsUsed); - bOrder = padOnesRight(bOrder, orderBitCount - bOrderBitsUsed); - } - - aOrder -= sOrder; - bOrder -= sOrder; - - if (_ordering.getOrder() == document::OrderingSpecification::DESCENDING) { - aOrder = -aOrder; - bOrder = -bOrder; - } - - if (aOrder == bOrder) { - return (a.toKey() < b.toKey()); // Reversed bucket id order - } - return (aOrder < bOrder); + return (a.toKey() < b.toKey()); // Reversed bucket id order } }; diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 88f3ad4f3c3..8a1f675a4c5 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -15,7 +15,6 @@ #include "memory_bounded_trace.h" #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/message/visitor.h> -#include <vespa/document/select/orderingspecification.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/visitorfactory.h> #include <vespa/documentapi/messagebus/messages/documentmessage.h> diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storageapi/src/vespa/storageapi/message/visitor.h index 4475e1c5614..f7dcaa63b20 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storageapi/src/vespa/storageapi/message/visitor.h @@ -13,7 +13,6 @@ #include <vespa/vdslib/container/visitorstatistics.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/messageapi/storagereply.h> -#include <vespa/document/select/orderingspecification.h> namespace storage::api { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index e5225f2569b..dc5dae9d516 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.yahoo.vespa.athenz.api.AthenzService; import java.time.Instant; -import java.util.Objects; import java.util.Set; /** @@ -88,26 +87,4 @@ public class SignedIdentityDocument { public IdentityType identityType() { return identityType; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SignedIdentityDocument that = (SignedIdentityDocument) o; - return signingKeyVersion == that.signingKeyVersion && - documentVersion == that.documentVersion && - Objects.equals(signature, that.signature) && - Objects.equals(providerUniqueId, that.providerUniqueId) && - Objects.equals(providerService, that.providerService) && - Objects.equals(configServerHostname, that.configServerHostname) && - Objects.equals(instanceHostname, that.instanceHostname) && - Objects.equals(createdAt, that.createdAt) && - Objects.equals(ipAddresses, that.ipAddresses) && - identityType == that.identityType; - } - - @Override - public int hashCode() { - return Objects.hash(signature, signingKeyVersion, providerUniqueId, providerService, documentVersion, configServerHostname, instanceHostname, createdAt, ipAddresses, identityType); - } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 7e572dae941..3b463e1af92 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -27,6 +27,7 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.log.LogLevel; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -238,6 +239,7 @@ public class RestApi extends LoggingRequestHandler { RestUri.apiErrorCodes.PARSER_ERROR); } catch (RuntimeException systemException) { + log.log(LogLevel.WARNING, "Internal runtime exception during Document V1 request handling", systemException); return Response.createErrorResponse(500, Exceptions.toMessageString(systemException), restUri, RestUri.apiErrorCodes.UNSPECIFIED); |