diff options
56 files changed, 358 insertions, 189 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3057be6251c..92d44c9e8e8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -31,8 +31,6 @@ public interface ModelContext { ApplicationPackage applicationPackage(); Optional<Model> previousModel(); Optional<ApplicationPackage> permanentApplicationPackage(); - // TODO: Remove after 7.338 has been released - default Optional<HostProvisioner> hostProvisioner() { return Optional.of(getHostProvisioner()); } HostProvisioner getHostProvisioner(); Provisioned provisioned(); DeployLogger deployLogger(); @@ -70,7 +68,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "jonmv"}) default double reindexerWindowSizeIncrement() { return 0.2; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"geirst"}, comment = "Remove when 7.342 is no longer in use") default boolean useDirectStorageApiRpc() { return true; } + @ModelFeatureFlag(owners = {"geirst"}, removeAfter = "7.342") default boolean useDirectStorageApiRpc() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 98cbd363bca..f8469aa6fa1 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -50,11 +50,6 @@ public class MockModelContext implements ModelContext { } @Override - public Optional<HostProvisioner> hostProvisioner() { - return Optional.empty(); - } - - @Override public HostProvisioner getHostProvisioner() { return DeployState.getDefaultModelHostProvisioner(applicationPackage); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 33f9d715801..e3e0edd7896 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -141,11 +141,6 @@ public class VespaModelFactoryTest { } @Override - public Optional<HostProvisioner> hostProvisioner() { - return Optional.of(provisionerToOverride); - } - - @Override public HostProvisioner getHostProvisioner() { return provisionerToOverride; } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index b998ed29b71..d27c585050b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import java.security.Principal; import java.security.PublicKey; +import java.time.Instant; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -31,9 +32,11 @@ import static java.util.Objects.requireNonNull; public abstract class LockedTenant { final TenantName name; + final Instant createdAt; - private LockedTenant(TenantName name) { + private LockedTenant(TenantName name, Instant createdAt) { this.name = requireNonNull(name); + this.createdAt = requireNonNull(createdAt); } static LockedTenant of(Tenant tenant, Lock lock) { @@ -61,8 +64,9 @@ public abstract class LockedTenant { private final Optional<PropertyId> propertyId; private final Optional<Contact> contact; - private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, Optional<Contact> contact) { - super(name); + private Athenz(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, + Optional<Contact> contact, Instant createdAt) { + super(name, createdAt); this.domain = domain; this.property = property; this.propertyId = propertyId; @@ -70,28 +74,28 @@ public abstract class LockedTenant { } private Athenz(AthenzTenant tenant) { - this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact()); + this(tenant.name(), tenant.domain(), tenant.property(), tenant.propertyId(), tenant.contact(), tenant.createdAt()); } @Override public AthenzTenant get() { - return new AthenzTenant(name, domain, property, propertyId, contact); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt); } public Athenz with(AthenzDomain domain) { - return new Athenz(name, domain, property, propertyId, contact); + return new Athenz(name, domain, property, propertyId, contact, createdAt); } public Athenz with(Property property) { - return new Athenz(name, domain, property, propertyId, contact); + return new Athenz(name, domain, property, propertyId, contact, createdAt); } public Athenz with(PropertyId propertyId) { - return new Athenz(name, domain, property, Optional.of(propertyId), contact); + return new Athenz(name, domain, property, Optional.of(propertyId), contact, createdAt); } public Athenz with(Contact contact) { - return new Athenz(name, domain, property, propertyId, Optional.of(contact)); + return new Athenz(name, domain, property, propertyId, Optional.of(contact), createdAt); } } @@ -104,20 +108,20 @@ public abstract class LockedTenant { private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; - private Cloud(TenantName name, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { - super(name); + private Cloud(TenantName name, Instant createdAt, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { + super(name, createdAt); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; } private Cloud(CloudTenant tenant) { - this(tenant.name(), Optional.empty(), tenant.developerKeys(), tenant.info()); + this(tenant.name(), tenant.createdAt(), Optional.empty(), tenant.developerKeys(), tenant.info()); } @Override public CloudTenant get() { - return new CloudTenant(name, creator, developerKeys, info); + return new CloudTenant(name, createdAt, creator, developerKeys, info); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -125,17 +129,17 @@ public abstract class LockedTenant { if (keys.containsKey(key)) throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); keys.put(key, principal); - return new Cloud(name, creator, keys, info); + return new Cloud(name, createdAt, creator, keys, info); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, creator, keys, info); + return new Cloud(name, createdAt, creator, keys, info); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, creator, developerKeys, newInfo); + return new Cloud(name, createdAt, creator, developerKeys, newInfo); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 5560796a97d..ffb1aae7299 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -99,7 +99,7 @@ public class TenantController { try (Lock lock = lock(tenantSpec.tenant())) { requireNonExistent(tenantSpec.tenant()); TenantId.validate(tenantSpec.tenant().value()); - curator.writeTenant(accessControl.createTenant(tenantSpec, credentials, asList())); + curator.writeTenant(accessControl.createTenant(tenantSpec, controller.clock().instant(), credentials, asList())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index c8cce94d479..3c035ef01ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -6,7 +6,6 @@ import com.google.common.cache.CacheLoader; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -33,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.ForbiddenException; +import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -41,6 +41,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Predicate; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -79,7 +80,7 @@ public class AthenzFacade implements AccessControl { } @Override - public Tenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { + public Tenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing) { AthenzTenantSpec spec = (AthenzTenantSpec) tenantSpec; AthenzCredentials athenzCredentials = (AthenzCredentials) credentials; AthenzDomain domain = spec.domain(); @@ -94,7 +95,8 @@ public class AthenzFacade implements AccessControl { AthenzTenant tenant = AthenzTenant.create(spec.tenant(), domain, spec.property(), - spec.propertyId()); + spec.propertyId(), + createdAt); if (existingWithSameDomain.isPresent()) { // Throw if domain is already taken. throw new IllegalArgumentException("Could not create tenant '" + spec.tenant().value() + @@ -123,11 +125,16 @@ public class AthenzFacade implements AccessControl { .filter(tenant -> tenant.type() == Tenant.Type.athenz && newDomain.equals(((AthenzTenant) tenant).domain())) .findAny(); + Instant createdAt = existing.stream() + .filter(tenant -> tenant.name().equals(spec.tenant())) + .findAny().orElseThrow() // Should not happen, we assert that the tenant exists before the method is called + .createdAt(); Tenant tenant = AthenzTenant.create(spec.tenant(), newDomain, spec.property(), - spec.propertyId()); + spec.propertyId(), + createdAt); if (existingWithSameDomain.isPresent()) { // Throw if domain taken by someone else, or do nothing if taken by this tenant. if ( ! existingWithSameDomain.get().equals(tenant)) // Equality by name. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 805803c13aa..e2d1c5bf7c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -52,6 +53,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -296,7 +298,10 @@ public class CuratorDb { } public Optional<Tenant> readTenant(TenantName name) { - return readSlime(tenantPath(name)).map(tenantSerializer::tenantFrom); + Supplier<Instant> tenantCreateTimeSupplier = () -> curator.getStat(tenantPath(name)) + .map(stat -> Instant.ofEpochMilli(stat.getCtime())) + .orElse(Instant.parse("2021-01-01T00:00:00Z")); + return readSlime(tenantPath(name)).map(bytes -> tenantSerializer.tenantFrom(bytes, tenantCreateTimeSupplier)); } public List<Tenant> readTenants() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 8b7590e88a9..79dbcc23299 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -26,9 +26,11 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfoBillingContact; import java.net.URI; import java.security.Principal; import java.security.PublicKey; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; /** * Slime serialization of {@link Tenant} sub-types. @@ -70,6 +72,7 @@ public class TenantSerializer { Cursor tenantObject = slime.setObject(); tenantObject.setString(nameField, tenant.name().value()); tenantObject.setString(typeField, valueOf(tenant.type())); + tenantObject.setLong(createdAtField, tenant.createdAt().toEpochMilli()); switch (tenant.type()) { case athenz: toSlime((AthenzTenant) tenant, tenantObject); break; @@ -113,33 +116,34 @@ public class TenantSerializer { billingInfoObject.setString(productCodeField, billingInfo.productCode()); } - public Tenant tenantFrom(Slime slime) { + public Tenant tenantFrom(Slime slime, Supplier<Instant> tenantCreateTimeSupplier) { Inspector tenantObject = slime.get(); - Tenant.Type type; - type = typeOf(tenantObject.field(typeField).asString()); + Tenant.Type type = typeOf(tenantObject.field(typeField).asString()); switch (type) { - case athenz: return athenzTenantFrom(tenantObject); - case cloud: return cloudTenantFrom(tenantObject); + case athenz: return athenzTenantFrom(tenantObject, tenantCreateTimeSupplier); + case cloud: return cloudTenantFrom(tenantObject, tenantCreateTimeSupplier); default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'."); } } - private AthenzTenant athenzTenantFrom(Inspector tenantObject) { + private AthenzTenant athenzTenantFrom(Inspector tenantObject, Supplier<Instant> tenantCreateTimeSupplier) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); AthenzDomain domain = new AthenzDomain(tenantObject.field(athenzDomainField).asString()); Property property = new Property(tenantObject.field(propertyField).asString()); Optional<PropertyId> propertyId = SlimeUtils.optionalString(tenantObject.field(propertyIdField)).map(PropertyId::new); Optional<Contact> contact = contactFrom(tenantObject.field(contactField)); - return new AthenzTenant(name, domain, property, propertyId, contact); + Instant createdAt = SlimeUtils.optionalLong(tenantObject.field(createdAtField)).map(Instant::ofEpochMilli).orElseGet(tenantCreateTimeSupplier); + return new AthenzTenant(name, domain, property, propertyId, contact, createdAt); } - private CloudTenant cloudTenantFrom(Inspector tenantObject) { + private CloudTenant cloudTenantFrom(Inspector tenantObject, Supplier<Instant> tenantCreateTimeSupplier) { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); + Instant createdAt = SlimeUtils.optionalLong(tenantObject.field(createdAtField)).map(Instant::ofEpochMilli).orElseGet(tenantCreateTimeSupplier); Optional<Principal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); - return new CloudTenant(name, creator, developerKeys, info); + return new CloudTenant(name, createdAt, creator, developerKeys, info); } private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8378b914fe4..13599664cef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1996,6 +1996,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<Instant> lastSubmission = applications.stream() .flatMap(app -> app.latestVersion().flatMap(ApplicationVersion::buildTime).stream()) .max(Comparator.naturalOrder()); + object.setLong("createdAtMillis", tenant.createdAt().toEpochMilli()); lastDev.ifPresent(instant -> object.setLong("lastDeploymentToDevMillis", instant.toEpochMilli())); lastSubmission.ifPresent(instant -> object.setLong("lastSubmissionToProdMillis", instant.toEpochMilli())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java index 02387213135..32bb866a5ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/AccessControl.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import java.time.Instant; import java.util.List; /** @@ -22,11 +23,12 @@ public interface AccessControl { * Sets up access control based on the given credentials, and returns a tenant, based on the given specification. * * @param tenantSpec specification for the tenant to create + * @param createdAt instant when the tenant was created * @param credentials the credentials for the entity requesting the creation * @param existing list of existing tenants, to check for conflicts * @return the created tenant, for keeping */ - Tenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing); + Tenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing); /** * Modifies access control based on the given credentials, and returns a modified tenant, based on the given specification. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index d37e1e05030..563c230e4f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.ForbiddenException; +import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -51,12 +52,12 @@ public class CloudAccessControl implements AccessControl { } @Override - public CloudTenant createTenant(TenantSpec tenantSpec, Credentials credentials, List<Tenant> existing) { + public CloudTenant createTenant(TenantSpec tenantSpec, Instant createdAt, Credentials credentials, List<Tenant> existing) { requireTenantCreationAllowed((Auth0Credentials) credentials); requireTenantTrialLimitNotReached(existing); CloudTenantSpec spec = (CloudTenantSpec) tenantSpec; - CloudTenant tenant = CloudTenant.create(spec.tenant(), credentials.user()); + CloudTenant tenant = CloudTenant.create(spec.tenant(), createdAt, credentials.user()); for (Role role : Roles.tenantRoles(spec.tenant())) { userManagement.createRole(role); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java index 5682d8b69fe..7c0e78337ee 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/AthenzTenant.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -23,11 +24,11 @@ public class AthenzTenant extends Tenant { /** * This should only be used by serialization. - * Use {@link #create(TenantName, AthenzDomain, Property, Optional)}. + * Use {@link #create(TenantName, AthenzDomain, Property, Optional, Instant)}. * */ public AthenzTenant(TenantName name, AthenzDomain domain, Property property, Optional<PropertyId> propertyId, - Optional<Contact> contact) { - super(name, Objects.requireNonNull(contact, "contact must be non-null")); + Optional<Contact> contact, Instant createdAt) { + super(name, createdAt, contact); this.domain = Objects.requireNonNull(domain, "domain must be non-null"); this.property = Objects.requireNonNull(property, "property must be non-null"); this.propertyId = Objects.requireNonNull(propertyId, "propertyId must be non-null"); @@ -60,13 +61,8 @@ public class AthenzTenant extends Tenant { /** Create a new Athenz tenant */ public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property, - Optional<PropertyId> propertyId) { - return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty()); - } - - public static AthenzTenant create(TenantName name, AthenzDomain domain, Property property, - Optional<PropertyId> propertyId, Optional<Contact> contact) { - return new AthenzTenant(requireName(name), domain, property, propertyId, contact); + Optional<PropertyId> propertyId, Instant createdAt) { + return new AthenzTenant(requireName(name), domain, property, propertyId, Optional.empty(), createdAt); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 67b285bb24f..2ed1ddc40e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.TenantName; import java.security.Principal; import java.security.PublicKey; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -22,16 +23,17 @@ public class CloudTenant extends Tenant { private final TenantInfo info; /** Public for the serialization layer — do not use! */ - public CloudTenant(TenantName name, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { - super(name, Optional.empty()); + public CloudTenant(TenantName name, Instant createdAt, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { + super(name, createdAt, Optional.empty()); this.creator = creator; this.developerKeys = developerKeys; this.info = Objects.requireNonNull(info); } /** Creates a tenant with the given name, provided it passes validation. */ - public static CloudTenant create(TenantName tenantName, Principal creator) { + public static CloudTenant create(TenantName tenantName, Instant createdAt, Principal creator) { return new CloudTenant(requireName(tenantName), + createdAt, Optional.ofNullable(creator), ImmutableBiMap.of(), TenantInfo.EMPTY); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java index b1dc0d8a5d5..7687796c697 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import java.time.Instant; import java.util.Objects; import java.util.Optional; @@ -15,10 +16,12 @@ import java.util.Optional; public abstract class Tenant { private final TenantName name; + private final Instant createdAt; private final Optional<Contact> contact; - Tenant(TenantName name, Optional<Contact> contact) { + Tenant(TenantName name, Instant createdAt, Optional<Contact> contact) { this.name = name; + this.createdAt = createdAt; this.contact = contact; } @@ -27,6 +30,11 @@ public abstract class Tenant { return name; } + /** Instant when the tenant was created */ + public Instant createdAt() { + return createdAt; + } + /** Contact information for this tenant */ public Optional<Contact> contact() { return contact; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 20caceee097..c26e4db7996 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -20,6 +20,7 @@ import org.junit.Test; import java.net.URI; import java.security.PublicKey; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -48,13 +49,15 @@ public class TenantSerializerTest { AthenzTenant tenant = AthenzTenant.create(TenantName.from("athenz-tenant"), new AthenzDomain("domain1"), new Property("property1"), - Optional.of(new PropertyId("1"))); - AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + Optional.of(new PropertyId("1")), + Instant.ofEpochMilli(1234L)); + AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.domain(), serialized.domain()); assertEquals(tenant.property(), serialized.property()); assertTrue(serialized.propertyId().isPresent()); assertEquals(tenant.propertyId(), serialized.propertyId()); + assertEquals(tenant.createdAt(), serialized.createdAt()); } @Test @@ -62,8 +65,9 @@ public class TenantSerializerTest { AthenzTenant tenant = AthenzTenant.create(TenantName.from("athenz-tenant"), new AthenzDomain("domain1"), new Property("property1"), - Optional.empty()); - AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + Optional.empty(), + Instant.EPOCH); + AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertFalse(serialized.propertyId().isPresent()); assertEquals(tenant.propertyId(), serialized.propertyId()); } @@ -74,32 +78,36 @@ public class TenantSerializerTest { new AthenzDomain("domain1"), new Property("property1"), Optional.of(new PropertyId("1")), - Optional.of(contact())); - AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + Optional.of(contact()), + Instant.EPOCH); + AthenzTenant serialized = (AthenzTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.contact(), serialized.contact()); } @Test public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.ofEpochMilli(1234L), Optional.of(new SimplePrincipal("foobar-user")), ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), otherPublicKey, new SimplePrincipal("jane")), TenantInfo.EMPTY); - CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.creator(), serialized.creator()); assertEquals(tenant.developerKeys(), serialized.developerKeys()); + assertEquals(tenant.createdAt(), serialized.createdAt()); } @Test public void cloud_tenant_with_info() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.EPOCH, Optional.of(new SimplePrincipal("foobar-user")), ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), otherPublicKey, new SimplePrincipal("jane")), TenantInfo.EMPTY.withName("Ofni Tnanet")); - CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant), () -> { throw new UnsupportedOperationException(); }); assertEquals(tenant.info(), serialized.info()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e5f11beb9a2..32f43f57152 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -92,7 +92,6 @@ import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static java.net.URLEncoder.encode; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.stream.Collectors.joining; @@ -1236,9 +1235,10 @@ public class ApplicationApiTest extends ControllerContainerTest { accessDenied, 403); - // Create legancy tenant name containing underscores + // Create legacy tenant name containing underscores tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, - new Property("property1"), Optional.empty(), Optional.empty())); + new Property("property1"), Optional.empty(), Optional.empty(), Instant.EPOCH)); + // POST (add) a Athenz tenant with dashes duplicates existing one with underscores tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .userIdentity(USER_ID) @@ -1339,7 +1339,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"athensDomain\":\"domain2\", \"property\":\"property1\"}") .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"tenant\":\"tenant1\",\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[],\"metaData\":{}}", + new File("tenant1.json"), 200); // Deleting a tenant for an Athens domain the user is not admin for is disallowed diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json index b11b65ead30..194f2cdd494 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application-with-metadata.json @@ -11,7 +11,8 @@ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ], - "metaData":{ + "metaData": { + "createdAtMillis": "(ignore)", "lastDeploymentToDevMillis":"(ignore)", "lastSubmissionToProdMillis":1000 } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index 578323be373..b02b1ea2565 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -11,5 +11,7 @@ "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ], - "metaData":{} + "metaData": { + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json index a7d1d7413f3..921eae72161 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-contact-info.json @@ -16,5 +16,7 @@ ] ], "applications": [], - "metaData":{} + "metaData": { + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json index 5e8f0e4b575..bd77a68a1eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json @@ -5,5 +5,7 @@ "property": "property2", "propertyId": "1234", "applications": [], - "metaData": {} + "metaData": { + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json index 82848fe971d..33ed505ce35 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json @@ -3,8 +3,8 @@ "type": "ATHENS", "athensDomain": "domain1", "property": "property1", - "applications": [ - - ], - "metaData":{} + "applications": [ ], + "metaData":{ + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json index 574c3d2c476..551b26c8513 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json @@ -6,5 +6,9 @@ "applications": [ @include(instance1-recursive.json) ], - "metaData":{"lastDeploymentToDevMillis":"(ignore)","lastSubmissionToProdMillis":1000} + "metaData": { + "createdAtMillis": "(ignore)", + "lastDeploymentToDevMillis": "(ignore)", + "lastSubmissionToProdMillis": 1000 + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json new file mode 100644 index 00000000000..a105a194974 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1.json @@ -0,0 +1,10 @@ +{ + "tenant": "tenant1", + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property1", + "applications": [], + "metaData": { + "createdAtMillis": "(ignore)" + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json index 21d5ceba805..7da5918419c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json @@ -16,5 +16,7 @@ ] ], "applications": [], - "metaData":{} + "metaData": { + "createdAtMillis": "(ignore)" + } }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 26c2e2d1175..d1f8fbebdb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -27,6 +27,7 @@ import java.net.URI; import java.net.http.HttpRequest; import java.security.PrivateKey; import java.security.PublicKey; +import java.time.Instant; import java.util.Optional; import java.util.Set; @@ -67,6 +68,7 @@ public class SignatureFilterTest { signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); tester.curator().writeTenant(new CloudTenant(appId.tenant(), + Instant.EPOCH, Optional.empty(), ImmutableBiMap.of(), TenantInfo.EMPTY)); @@ -107,6 +109,7 @@ public class SignatureFilterTest { // Signed request gets a developer role when a matching developer key is stored for the tenant. tester.curator().writeTenant(new CloudTenant(appId.tenant(), + Instant.EPOCH, Optional.empty(), ImmutableBiMap.of(publicKey, () -> "user"), TenantInfo.EMPTY)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 28dad0b0c2f..9323067904c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -16,5 +16,7 @@ "clusterSize": 5 }, "applications": [], - "metaData":{} + "metaData":{ + "createdAtMillis": "(ignore)" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index 73b97e09827..eaabb9fe3e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -9,5 +9,7 @@ "clusterSize": 5 }, "applications": [], - "metaData":{} + "metaData":{ + "createdAtMillis": "(ignore)" + } } diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 00618f2c838..3c94639d0aa 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -56,7 +56,7 @@ vespa_define_module( src/tests/instruction/add_trivial_dimension_optimizer src/tests/instruction/fast_rename_optimizer src/tests/instruction/dense_inplace_join_function - src/tests/instruction/dense_pow_as_map_optimizer + src/tests/instruction/pow_as_map_optimizer src/tests/instruction/remove_trivial_dimension_optimizer src/tests/instruction/dense_replace_type_function src/tests/instruction/dense_simple_join_function diff --git a/eval/src/tests/eval/aggr/aggr_test.cpp b/eval/src/tests/eval/aggr/aggr_test.cpp index 9045df68305..5eddb026406 100644 --- a/eval/src/tests/eval/aggr/aggr_test.cpp +++ b/eval/src/tests/eval/aggr/aggr_test.cpp @@ -85,8 +85,9 @@ TEST("require that PROD aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::PROD); } -TEST("require that Prod combine works as expected") { +TEST("require that Prod static API works as expected") { using Type = Prod<double>; + EXPECT_EQUAL(Type::null_value(), 1.0); EXPECT_EQUAL(Type::combine(3,7), 21.0); EXPECT_EQUAL(Type::combine(5,4), 20.0); } @@ -103,8 +104,9 @@ TEST("require that SUM aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::SUM); } -TEST("require that Sum combine works as expected") { +TEST("require that Sum static API works as expected") { using Type = Sum<double>; + EXPECT_EQUAL(Type::null_value(), 0.0); EXPECT_EQUAL(Type::combine(3,7), 10.0); EXPECT_EQUAL(Type::combine(5,4), 9.0); } @@ -121,8 +123,10 @@ TEST("require that MAX aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::MAX); } -TEST("require that Max combine works as expected") { +TEST("require that Max static API works as expected") { using Type = Max<double>; + EXPECT_EQUAL(Max<double>::null_value(), -std::numeric_limits<double>::infinity()); + EXPECT_EQUAL(Max<float>::null_value(), -std::numeric_limits<float>::infinity()); EXPECT_EQUAL(Type::combine(3,7), 7.0); EXPECT_EQUAL(Type::combine(5,4), 5.0); } @@ -165,8 +169,10 @@ TEST("require that MIN aggregator works as expected") { EXPECT_TRUE(aggr.enum_value() == Aggr::MIN); } -TEST("require that Min combine works as expected") { +TEST("require that Min static API works as expected") { using Type = Min<double>; + EXPECT_EQUAL(Min<double>::null_value(), std::numeric_limits<double>::infinity()); + EXPECT_EQUAL(Min<float>::null_value(), std::numeric_limits<float>::infinity()); EXPECT_EQUAL(Type::combine(3,7), 3.0); EXPECT_EQUAL(Type::combine(5,4), 4.0); } diff --git a/eval/src/tests/instruction/dense_pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/instruction/dense_pow_as_map_optimizer/CMakeLists.txt deleted file mode 100644 index d6ce9f1924c..00000000000 --- a/eval/src/tests/instruction/dense_pow_as_map_optimizer/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(eval_dense_pow_as_map_optimizer_test_app TEST - SOURCES - dense_pow_as_map_optimizer_test.cpp - DEPENDS - vespaeval - GTest::GTest -) -vespa_add_test(NAME eval_dense_pow_as_map_optimizer_test_app COMMAND eval_dense_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp index 3ab971dd34d..9e2090fa968 100644 --- a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp +++ b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp @@ -32,7 +32,8 @@ std::vector<Layout> layouts = { float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), {x(3),y({"foo", "bar"}),z(7)}, {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) + float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}), + {x(3),y({}),z(7)} }; TensorSpec perform_generic_reduce(const TensorSpec &a, Aggr aggr, const std::vector<vespalib::string> &dims, @@ -69,7 +70,9 @@ TEST(GenericReduceTest, sparse_reduce_plan_can_be_created) { void test_generic_reduce_with(const ValueBuilderFactory &factory) { for (const Layout &layout: layouts) { TensorSpec input = spec(layout, Div16(N())); + SCOPED_TRACE(fmt("tensor type: %s, num_cells: %zu", input.type().c_str(), input.cells().size())); for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { + SCOPED_TRACE(fmt("aggregator: %s", AggrNames::name_of(aggr)->c_str())); for (const Domain &domain: layout) { auto expect = ReferenceOperations::reduce(input, aggr, {domain.dimension}).normalize(); auto actual = perform_generic_reduce(input, aggr, {domain.dimension}, factory); diff --git a/eval/src/tests/instruction/pow_as_map_optimizer/CMakeLists.txt b/eval/src/tests/instruction/pow_as_map_optimizer/CMakeLists.txt new file mode 100644 index 00000000000..25eeb73df37 --- /dev/null +++ b/eval/src/tests/instruction/pow_as_map_optimizer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_pow_as_map_optimizer_test_app TEST + SOURCES + pow_as_map_optimizer_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_pow_as_map_optimizer_test_app COMMAND eval_pow_as_map_optimizer_test_app) diff --git a/eval/src/tests/instruction/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp index 67567b4e289..920198f67d7 100644 --- a/eval/src/tests/instruction/dense_pow_as_map_optimizer/dense_pow_as_map_optimizer_test.cpp +++ b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp @@ -19,7 +19,7 @@ EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() .add("a", spec(1.5)) .add("b", spec(2.5)) - .add("sparse", spec({x({"a"})}, N())) + .add("sparse", spec({x({"a","b"})}, N())) .add("mixed", spec({x({"a"}),y(5)}, N())) .add_matrix("x", 5, "y", 3); } @@ -30,11 +30,10 @@ void verify_optimized(const vespalib::string &expr, op1_t op1, bool inplace = fa EvalFixture fixture(prod_factory, expr, param_repo, true, true); EXPECT_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); EXPECT_EQ(fixture.result(), slow_fixture.result()); - auto info = fixture.find_all<DenseSimpleMapFunction>(); + auto info = fixture.find_all<tensor_function::Map>(); ASSERT_EQ(info.size(), 1u); EXPECT_TRUE(info[0]->result_is_mutable()); EXPECT_EQ(info[0]->function(), op1); - EXPECT_EQ(info[0]->inplace(), inplace); ASSERT_EQ(fixture.num_params(), 1); if (inplace) { EXPECT_EQ(fixture.get_param(0), fixture.result()); @@ -76,16 +75,16 @@ TEST(PowAsMapTest, hypercubed_dense_tensor_is_not_optimized) { verify_not_optimized("join(x5y3,4.0,f(x,y)(pow(x,y)))"); } -TEST(PowAsMapTest, scalar_join_is_not_optimized) { - verify_not_optimized("join(a,2.0,f(x,y)(pow(x,y)))"); +TEST(PowAsMapTest, scalar_join_is_optimized) { + verify_optimized("join(a,2.0,f(x,y)(pow(x,y)))", Square::f); } -TEST(PowAsMapTest, sparse_join_is_not_optimized) { - verify_not_optimized("join(sparse,2.0,f(x,y)(pow(x,y)))"); +TEST(PowAsMapTest, sparse_join_is_optimized) { + verify_optimized("join(sparse,2.0,f(x,y)(pow(x,y)))", Square::f); } -TEST(PowAsMapTest, mixed_join_is_not_optimized) { - verify_not_optimized("join(mixed,2.0,f(x,y)(pow(x,y)))"); +TEST(PowAsMapTest, mixed_join_is_optimized) { + verify_optimized("join(mixed,2.0,f(x,y)(pow(x,y)))", Square::f); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 89bddfda933..3345d7dc8ee 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -814,6 +814,14 @@ TEST(DenseJoin, partial_overlap) { benchmark_join("dense partial overlap multiply", lhs, rhs, operation::Mul::f); } +TEST(DenseJoin, subset_overlap) { + auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto rhs_inner = make_matrix(D::idx("b", 16), D::idx("c", 16), 2.0); + auto rhs_outer = make_matrix(D::idx("a", 16), D::idx("b", 16), 3.0); + benchmark_join("dense subset overlap inner multiply", lhs, rhs_inner, operation::Mul::f); + benchmark_join("dense subset overlap outer multiply", lhs, rhs_outer, operation::Mul::f); +} + TEST(DenseJoin, no_overlap) { auto lhs = make_cube(D::idx("a", 4), D::idx("e", 4), D::idx("f", 4), 1.0); auto rhs = make_cube(D::idx("b", 4), D::idx("c", 4), D::idx("d", 4), 2.0); diff --git a/eval/src/vespa/eval/eval/aggr.h b/eval/src/vespa/eval/eval/aggr.h index d2e7a64b51b..516ead0f0bf 100644 --- a/eval/src/vespa/eval/eval/aggr.h +++ b/eval/src/vespa/eval/eval/aggr.h @@ -61,9 +61,10 @@ struct Aggregator { namespace aggr { -// can we start by picking any value from the set to be reduced and -// use the templated aggregator 'combine' function in arbitrary order -// to end up with (approximately) the correct result? +// can we start by picking any value from the set to be reduced (or +// the special aggregator-specific null_value) and use the templated +// aggregator 'combine' function in arbitrary order to end up with +// (approximately) the correct result? constexpr bool is_simple(Aggr aggr) { return ((aggr == Aggr::PROD) || (aggr == Aggr::SUM) || @@ -124,12 +125,13 @@ private: T _prod; public: using value_type = T; - constexpr Prod() : _prod{1} {} + constexpr Prod() : _prod{null_value()} {} constexpr Prod(T value) : _prod{value} {} - constexpr void sample(T value) { _prod *= value; } - constexpr void merge(const Prod &rhs) { _prod *= rhs._prod; } + constexpr void sample(T value) { _prod = combine(_prod, value); } + constexpr void merge(const Prod &rhs) { _prod = combine(_prod, rhs._prod); } constexpr T result() const { return _prod; } static constexpr Aggr enum_value() { return Aggr::PROD; } + static constexpr T null_value() { return 1; } static constexpr T combine(T a, T b) { return (a * b); } }; @@ -138,12 +140,13 @@ private: T _sum; public: using value_type = T; - constexpr Sum() : _sum{0} {} + constexpr Sum() : _sum{null_value()} {} constexpr Sum(T value) : _sum{value} {} - constexpr void sample(T value) { _sum += value; } - constexpr void merge(const Sum &rhs) { _sum += rhs._sum; } + constexpr void sample(T value) { _sum = combine(_sum, value); } + constexpr void merge(const Sum &rhs) { _sum = combine(_sum, rhs._sum); } constexpr T result() const { return _sum; } static constexpr Aggr enum_value() { return Aggr::SUM; } + static constexpr T null_value() { return 0; } static constexpr T combine(T a, T b) { return (a + b); } }; @@ -152,12 +155,13 @@ private: T _max; public: using value_type = T; - constexpr Max() : _max{-std::numeric_limits<T>::infinity()} {} + constexpr Max() : _max{null_value()} {} constexpr Max(T value) : _max{value} {} - constexpr void sample(T value) { _max = std::max(_max, value); } - constexpr void merge(const Max &rhs) { _max = std::max(_max, rhs._max); } + constexpr void sample(T value) { _max = combine(_max, value); } + constexpr void merge(const Max &rhs) { _max = combine(_max, rhs._max); } constexpr T result() const { return _max; } static constexpr Aggr enum_value() { return Aggr::MAX; } + static constexpr T null_value() { return -std::numeric_limits<T>::infinity(); } static constexpr T combine(T a, T b) { return std::max(a,b); } }; @@ -204,12 +208,13 @@ private: T _min; public: using value_type = T; - constexpr Min() : _min{std::numeric_limits<T>::infinity()} {} + constexpr Min() : _min{null_value()} {} constexpr Min(T value) : _min{value} {} - constexpr void sample(T value) { _min = std::min(_min, value); } - constexpr void merge(const Min &rhs) { _min = std::min(_min, rhs._min); } + constexpr void sample(T value) { _min = combine(_min, value); } + constexpr void merge(const Min &rhs) { _min = combine(_min, rhs._min); } constexpr T result() const { return _min; } static constexpr Aggr enum_value() { return Aggr::MIN; } + static constexpr T null_value() { return std::numeric_limits<T>::infinity(); } static constexpr T combine(T a, T b) { return std::min(a,b); } }; diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index 515b48b3693..02d6bdbf0f3 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -16,7 +16,7 @@ #include <vespa/eval/instruction/dense_simple_expand_function.h> #include <vespa/eval/instruction/dense_simple_join_function.h> #include <vespa/eval/instruction/join_with_number_function.h> -#include <vespa/eval/instruction/dense_pow_as_map_optimizer.h> +#include <vespa/eval/instruction/pow_as_map_optimizer.h> #include <vespa/eval/instruction/dense_simple_map_function.h> #include <vespa/eval/instruction/vector_from_doubles_function.h> #include <vespa/eval/instruction/dense_tensor_create_function.h> @@ -65,7 +65,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &factory, c child.set(DenseTensorPeekFunction::optimize(child.get(), stash)); child.set(DenseLambdaPeekOptimizer::optimize(child.get(), stash)); child.set(FastRenameOptimizer::optimize(child.get(), stash)); - child.set(DensePowAsMapOptimizer::optimize(child.get(), stash)); + child.set(PowAsMapOptimizer::optimize(child.get(), stash)); child.set(DenseSimpleMapFunction::optimize(child.get(), stash)); child.set(DenseSimpleJoinFunction::optimize(child.get(), stash)); child.set(JoinWithNumberFunction::optimize(child.get(), stash)); diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index 1317a2c2cf6..452ed51610a 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -9,7 +9,7 @@ vespa_add_library(eval_instruction OBJECT dense_lambda_peek_optimizer.cpp dense_matmul_function.cpp dense_multi_matmul_function.cpp - dense_pow_as_map_optimizer.cpp + pow_as_map_optimizer.cpp remove_trivial_dimension_optimizer.cpp dense_simple_expand_function.cpp dense_simple_join_function.cpp diff --git a/eval/src/vespa/eval/instruction/generic_reduce.cpp b/eval/src/vespa/eval/instruction/generic_reduce.cpp index b6393d0d713..2d4144d64b1 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.cpp +++ b/eval/src/vespa/eval/instruction/generic_reduce.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/util/visit_ranges.h> +#include <algorithm> #include <cassert> #include <array> @@ -48,7 +49,7 @@ struct SparseReduceState { std::vector<string_id> full_address; std::vector<string_id*> fetch_address; std::vector<string_id*> keep_address; - size_t subspace; + size_t subspace; SparseReduceState(const SparseReducePlan &plan) : full_address(plan.keep_dims.size() + plan.num_reduce_dims), @@ -72,8 +73,8 @@ Value::UP generic_reduce(const Value &value, const ReduceParam ¶m) { auto cells = value.cells().typify<ICT>(); ArrayArrayMap<string_id,AGGR> map(param.sparse_plan.keep_dims.size(), - param.dense_plan.out_size, - value.index().size()); + param.dense_plan.out_size, + value.index().size()); SparseReduceState sparse(param.sparse_plan); auto full_view = value.index().create_view({}); full_view->lookup({}); @@ -94,9 +95,7 @@ generic_reduce(const Value &value, const ReduceParam ¶m) { }); if ((map.size() == 0) && param.sparse_plan.keep_dims.empty()) { auto zero = builder->add_subspace(); - for (size_t i = 0; i < zero.size(); ++i) { - zero[i] = OCT{}; - } + std::fill(zero.begin(), zero.end(), OCT{}); } return builder->build(std::move(builder)); } @@ -109,6 +108,50 @@ void my_generic_reduce_op(State &state, uint64_t param_in) { auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(up)); const Value &result_ref = *(result.get()); state.pop_push(result_ref); +} + +template <typename ICT, typename OCT, typename AGGR, bool forward_index> +void my_generic_dense_reduce_op(State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<ReduceParam>(param_in); + const Value &value = state.peek(0); + auto cells = value.cells().typify<ICT>(); + const auto &index = value.index(); + size_t num_subspaces = index.size(); + size_t out_cells_size = forward_index ? (param.dense_plan.out_size * num_subspaces) : param.dense_plan.out_size; + auto out_cells = state.stash.create_uninitialized_array<OCT>(out_cells_size); + if (num_subspaces > 0) { + if constexpr (aggr::is_simple(AGGR::enum_value())) { + OCT *dst = out_cells.begin(); + std::fill(out_cells.begin(), out_cells.end(), AGGR::null_value()); + auto combine = [&](size_t src_idx, size_t dst_idx) { dst[dst_idx] = AGGR::combine(dst[dst_idx], cells[src_idx]); }; + for (size_t i = 0; i < num_subspaces; ++i) { + param.dense_plan.execute(i * param.dense_plan.in_size, combine); + if (forward_index) { + dst += param.dense_plan.out_size; + } + } + } else { + std::vector<AGGR> aggr_state(out_cells_size); + AGGR *dst = &aggr_state[0]; + auto sample = [&](size_t src_idx, size_t dst_idx) { dst[dst_idx].sample(cells[src_idx]); }; + for (size_t i = 0; i < num_subspaces; ++i) { + param.dense_plan.execute(i * param.dense_plan.in_size, sample); + if (forward_index) { + dst += param.dense_plan.out_size; + } + } + for (size_t i = 0; i < aggr_state.size(); ++i) { + out_cells[i] = aggr_state[i].result(); + } + } + } else if (!forward_index) { + std::fill(out_cells.begin(), out_cells.end(), OCT{}); + } + if (forward_index) { + state.pop_push(state.stash.create<ValueView>(param.res_type, index, TypedCells(out_cells))); + } else { + state.pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells))); + } }; template <typename ICT, typename OCT, typename AGGR> @@ -147,10 +190,17 @@ void my_full_reduce_op(State &state, uint64_t) { struct SelectGenericReduceOp { template <typename ICT, typename OCT, typename AGGR> static auto invoke(const ReduceParam ¶m) { + using AggrType = typename AGGR::template templ<OCT>; if (param.res_type.is_scalar()) { - return my_full_reduce_op<ICT, OCT, typename AGGR::template templ<OCT>>; + return my_full_reduce_op<ICT, OCT, AggrType>; + } + if (param.sparse_plan.should_forward_index()) { + return my_generic_dense_reduce_op<ICT, OCT, AggrType, true>; } - return my_generic_reduce_op<ICT, OCT, typename AGGR::template templ<OCT>>; + if (param.res_type.is_dense()) { + return my_generic_dense_reduce_op<ICT, OCT, AggrType, false>; + } + return my_generic_reduce_op<ICT, OCT, AggrType>; } }; @@ -227,6 +277,12 @@ SparseReducePlan::SparseReducePlan(const ValueType &type, const ValueType &res_t } } +bool +SparseReducePlan::should_forward_index() const +{ + return ((num_reduce_dims == 0) && (!keep_dims.empty())); +} + SparseReducePlan::~SparseReducePlan() = default; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/instruction/generic_reduce.h b/eval/src/vespa/eval/instruction/generic_reduce.h index f753a3e51cd..5faafb0325d 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.h +++ b/eval/src/vespa/eval/instruction/generic_reduce.h @@ -30,6 +30,7 @@ struct DenseReducePlan { struct SparseReducePlan { size_t num_reduce_dims; std::vector<size_t> keep_dims; + bool should_forward_index() const; SparseReducePlan(const ValueType &type, const ValueType &res_type); ~SparseReducePlan(); }; diff --git a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.cpp b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.cpp index 61ef2243480..00fbc1c642b 100644 --- a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.cpp +++ b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.cpp @@ -1,6 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dense_pow_as_map_optimizer.h" +#include "pow_as_map_optimizer.h" #include "dense_simple_map_function.h" #include <vespa/eval/eval/operation.h> @@ -10,14 +10,13 @@ using namespace tensor_function; using namespace operation; const TensorFunction & -DensePowAsMapOptimizer::optimize(const TensorFunction &expr, Stash &stash) +PowAsMapOptimizer::optimize(const TensorFunction &expr, Stash &stash) { if (auto join = as<Join>(expr)) { const TensorFunction &lhs = join->lhs(); const TensorFunction &rhs = join->rhs(); if ((join->function() == Pow::f) && - lhs.result_type().is_dense() && - rhs.result_type().is_double()) + rhs.result_type().is_scalar()) { if (auto const_value = as<ConstValue>(rhs)) { if (const_value->value().as_double() == 2.0) { diff --git a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.h b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.h index e61069b87b0..df9735f76ca 100644 --- a/eval/src/vespa/eval/instruction/dense_pow_as_map_optimizer.h +++ b/eval/src/vespa/eval/instruction/pow_as_map_optimizer.h @@ -10,9 +10,8 @@ namespace vespalib::eval { * Tensor function optimizer for converting join expressions on the * form 'join(tensor,<small integer constant>,f(x,y)(pow(x,y))' to * expressions on the form 'map(tensor,f(x)(x*x...))'. - * TODO: extend to mixed tensors. **/ -struct DensePowAsMapOptimizer { +struct PowAsMapOptimizer { static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); }; diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 3237dede24a..81a9484e98c 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -15,7 +15,7 @@ namespace vespalib { class IDestructorCallback; } namespace storage::spi { class IResourceUsageListener; -class BucketExecutor; +struct BucketExecutor; /** * This interface is the basis for a persistence provider in Vespa. A diff --git a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp index 73fac7d9439..a1e713802c1 100644 --- a/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_scan_iterator/document_scan_iterator_test.cpp @@ -20,10 +20,10 @@ typedef std::vector<uint32_t> LidVector; struct Fixture { DocumentMetaStore _metaStore; - DocumentScanIterator _itr; + std::unique_ptr<DocumentScanIterator> _itr; Fixture() : _metaStore(std::make_shared<BucketDBOwner>()), - _itr(_metaStore) + _itr() { _metaStore.constructFreeList(); } @@ -43,17 +43,24 @@ struct Fixture return *this; } LidSet scan(uint32_t count, uint32_t compactLidLimit, uint32_t maxDocsToScan = 10) { + if (!_itr) { + _itr = std::make_unique<DocumentScanIterator>(_metaStore); + } LidSet retval; for (uint32_t i = 0; i < count; ++i) { - retval.insert(next(compactLidLimit, maxDocsToScan, false)); - EXPECT_TRUE(_itr.valid()); + uint32_t lid = next(compactLidLimit, maxDocsToScan, false); + retval.insert(lid); + EXPECT_TRUE(_itr->valid() || lid <= compactLidLimit); } EXPECT_EQUAL(0u, next(compactLidLimit, maxDocsToScan, false)); - EXPECT_FALSE(_itr.valid()); + EXPECT_FALSE(_itr->valid()); return retval; } uint32_t next(uint32_t compactLidLimit, uint32_t maxDocsToScan = 10, bool retry = false) { - return _itr.next(compactLidLimit, maxDocsToScan, retry).lid; + if (!_itr) { + _itr = std::make_unique<DocumentScanIterator>(_metaStore); + } + return _itr->next(compactLidLimit, maxDocsToScan, retry).lid; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp index cbab4b72971..5f25767368a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.cpp @@ -11,8 +11,7 @@ typedef IDocumentMetaStore::Iterator Iterator; DocumentScanIterator::DocumentScanIterator(const IDocumentMetaStore &metaStore) : _metaStore(metaStore), - _lastGid(), - _lastGidValid(false), + _lastLid(_metaStore.getCommittedDocIdLimit()), _itrValid(true) { } @@ -24,27 +23,19 @@ DocumentScanIterator::valid() const } DocumentMetaData -DocumentScanIterator::next(uint32_t compactLidLimit, - uint32_t maxDocsToScan, - bool retry) +DocumentScanIterator::next(uint32_t compactLidLimit, uint32_t maxDocsToScan, bool retry) { - Iterator itr = (_lastGidValid ? - (retry ? _metaStore.lowerBound(_lastGid) : _metaStore.upperBound(_lastGid)) - : _metaStore.begin()); - uint32_t i = 1; // We have already 'scanned' a document when creating the iterator - for (; i < maxDocsToScan && itr.valid() && itr.getKey().get_lid() <= compactLidLimit; ++i, ++itr) {} - if (itr.valid()) { - uint32_t lid = itr.getKey().get_lid(); - const RawDocumentMetaData &metaData = _metaStore.getRawMetaData(lid); - _lastGid = metaData.getGid(); - _lastGidValid = true; - if (lid > compactLidLimit) { - return DocumentMetaData(lid, metaData.getTimestamp(), - metaData.getBucketId(), metaData.getGid()); + if (!retry) { + --_lastLid; + } + for (uint32_t i(0); (_lastLid > compactLidLimit) && (i < maxDocsToScan); ++i, --_lastLid) { + if (_metaStore.validLid(_lastLid)) { + const RawDocumentMetaData &metaData = _metaStore.getRawMetaData(_lastLid); + return DocumentMetaData(_lastLid, metaData.getTimestamp(), + metaData.getBucketId(), metaData.getGid()); } - } else { - _itrValid = false; } + _itrValid = (_lastLid > compactLidLimit) ; return DocumentMetaData(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h index fec547236d2..93d9106ae87 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_scan_iterator.h @@ -15,19 +15,14 @@ class DocumentScanIterator : public IDocumentScanIterator { private: const IDocumentMetaStore &_metaStore; - document::GlobalId _lastGid; - bool _lastGidValid; + uint32_t _lastLid; bool _itrValid; public: DocumentScanIterator(const IDocumentMetaStore &_metaStore); - // Implements IDocumentScanIterator - virtual bool valid() const override; - - virtual search::DocumentMetaData next(uint32_t compactLidLimit, - uint32_t maxDocsToScan, - bool retry) override; + bool valid() const override; + search::DocumentMetaData next(uint32_t compactLidLimit, uint32_t maxDocsToScan, bool retry) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 0e6d20115a0..4396224fb01 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -31,7 +31,7 @@ namespace vespalib { class StateServer; } namespace search::transactionlog { class TransLogServerApp; } namespace metrics { class MetricLockGuard; } -namespace storage::spi { class PersistenceProvider; } +namespace storage::spi { struct PersistenceProvider; } namespace proton { diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 433f535546f..3742879bd30 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -35,7 +35,7 @@ namespace storage { -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } class StorageBucketDBInitializer; DEFINE_PRIMITIVE_WRAPPER(uint16_t, NodeIndex); diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 1518c8594aa..924678a6cd0 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -151,6 +151,12 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil { return cb; } + void set_up_distributor_with_feed_blocked_state() { + setup_distributor(2, 2, + lib::ClusterStateBundle(lib::ClusterState("distributor:1 storage:2"), + {}, {true, "full disk"}, false)); + } + }; TwoPhaseUpdateOperationTest::TwoPhaseUpdateOperationTest() = default; @@ -1091,6 +1097,17 @@ TEST_F(TwoPhaseUpdateOperationTest, update_gets_are_sent_with_strong_consistency EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Strong); } +TEST_F(TwoPhaseUpdateOperationTest, operation_is_rejected_in_safe_path_if_feed_is_blocked) { + set_up_distributor_with_feed_blocked_state(); + auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas to trigger safe path + cb->start(_sender, framework::MilliSecTime(0)); + + EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(NO_SPACE, External feed is blocked due to resource exhaustion: full disk)", + _sender.getLastReply(true)); +} + struct ThreePhaseUpdateTest : TwoPhaseUpdateOperationTest {}; TEST_F(ThreePhaseUpdateTest, metadata_only_gets_are_sent_if_3phase_update_enabled) { diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 80e7942c68e..362acdf18ec 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "twophaseupdateoperation.h" #include "getoperation.h" #include "putoperation.h" +#include "twophaseupdateoperation.h" #include "updateoperation.h" -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/distributor_bucket_space_repo.h> -#include <vespa/storageapi/message/persistence.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/select/parser.h> -#include <vespa/document/fieldset/fieldsets.h> +#include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/storage/distributor/distributor_bucket_space_repo.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> @@ -197,6 +198,10 @@ TwoPhaseUpdateOperation::startFastPathUpdate(DistributorMessageSender& sender, s void TwoPhaseUpdateOperation::startSafePathUpdate(DistributorMessageSender& sender) { + if (_op_ctx.cluster_state_bundle().block_feed_in_cluster()) { + send_feed_blocked_error_reply(sender); + return; + } _mode = Mode::SLOW_PATH; auto get_operation = create_initial_safe_path_get_operation(); GetOperation& op = *get_operation; @@ -279,6 +284,15 @@ TwoPhaseUpdateOperation::sendLostOwnershipTransientErrorReply(DistributorMessage } void +TwoPhaseUpdateOperation::send_feed_blocked_error_reply(DistributorMessageSender& sender) +{ + sendReplyWithResult(sender, + api::ReturnCode(api::ReturnCode::NO_SPACE, + "External feed is blocked due to resource exhaustion: " + + _op_ctx.cluster_state_bundle().feed_block()->description())); +} + +void TwoPhaseUpdateOperation::schedulePutsWithUpdatedDocument(std::shared_ptr<document::Document> doc, api::Timestamp putTimestamp, DistributorMessageSender& sender) { diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h index af45932b530..d353498c8e3 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h @@ -104,6 +104,7 @@ private: void startSafePathUpdate(DistributorMessageSender&); bool lostBucketOwnershipBetweenPhases() const; void sendLostOwnershipTransientErrorReply(DistributorMessageSender&); + void send_feed_blocked_error_reply(DistributorMessageSender& sender); void schedulePutsWithUpdatedDocument( std::shared_ptr<document::Document>, api::Timestamp, diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.h b/storage/src/vespa/storage/persistence/bucketprocessor.h index 5b96696475e..c4ebcce6165 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.h +++ b/storage/src/vespa/storage/persistence/bucketprocessor.h @@ -11,7 +11,7 @@ #include <vespa/persistence/spi/context.h> namespace document { class FieldSet; } -namespace storage::spi { class PersistenceProvider; } +namespace storage::spi { struct PersistenceProvider; } namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 489dd1f97e9..b69657f2692 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -36,7 +36,7 @@ namespace api { class StorageReply; class BucketCommand; } -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } struct FileStorManagerTest; class ReadBucketList; diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 99133073a19..b60d3fd8d5d 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -18,7 +18,7 @@ namespace storage::api { } namespace storage::spi { - class PersistenceProvider; + struct PersistenceProvider; } namespace storage { diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.h b/storage/src/vespa/storage/persistence/splitbitdetector.h index d2c415fc526..22261c03f46 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.h +++ b/storage/src/vespa/storage/persistence/splitbitdetector.h @@ -23,7 +23,7 @@ namespace storage { -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } struct SplitBitDetector { diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index f75cf867327..ef835ec9f8a 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -17,7 +17,7 @@ namespace storage { -namespace spi { class PersistenceProvider; } +namespace spi { struct PersistenceProvider; } class FileStorManager; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 7b13332052f..884aa954599 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -122,7 +122,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.clock = clock; this.identity = new AthenzService(config.domain(), config.service()); this.ztsEndpoint = URI.create(config.ztsUrl()); - roleSslCertCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::requestRoleCertificate); + roleSslCertCache = crateAutoReloadableCache(ROLE_SSL_CONTEXT_EXPIRY, this::requestRoleCertificate, this.scheduler); roleKeyManagerCache = new HashMap<>(); roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); @@ -145,6 +145,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen }); } + private static <KEY, VALUE> LoadingCache<KEY, VALUE> crateAutoReloadableCache(Duration expiry, Function<KEY, VALUE> cacheLoader, ScheduledExecutorService scheduler) { + LoadingCache<KEY, VALUE> cache = createCache(expiry, cacheLoader); + + // The cache above will reload it's contents if and only if a request for the key is made. Scheduling + // a cache reloader to reload all keys in this cache. + scheduler.scheduleAtFixedRate(() -> { cache.asMap().keySet().forEach(cache::getUnchecked);}, + expiry.dividedBy(4).toMinutes(), + expiry.dividedBy(4).toMinutes(), + TimeUnit.MINUTES); + return cache; + } + private static SSLContext createIdentitySslContext(X509ExtendedKeyManager keyManager, Path trustStore) { return new SslContextBuilder() .withKeyManager(keyManager) |