diff options
38 files changed, 480 insertions, 248 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 04c82e3c572..2a48c1b128e 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -5493,9 +5493,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^4: - version "4.4.10" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.4.10.tgz#3794639cc433f7cb33ad286930bf0378c86261c8" - integrity sha512-TzIjiqx9BEXF8yzYdF2NTf1kFFbjMjUSV0LFZ3HyHoI3SGSPLnnFUKiIQtL3gl2AjHvMrprOvQ3amzaHgQlAxw== + version "4.4.11" + resolved "https://registry.yarnpkg.com/vite/-/vite-4.4.11.tgz#babdb055b08c69cfc4c468072a2e6c9ca62102b0" + integrity sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A== dependencies: esbuild "^0.18.10" postcss "^8.4.27" diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 2c5be906633..b28401f1873 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1289,7 +1289,8 @@ "public boolean useReconfigurableDispatcher()", "public int contentLayerMetadataFeatureLevel()", "public boolean dynamicHeapSize()", - "public java.lang.String unknownConfigDefinition()" + "public java.lang.String unknownConfigDefinition()", + "public int searchHandlerThreadpool()" ], "fields" : [ ] }, diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 57d013ebd01..024a4c233e5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -120,6 +120,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "log"; } + @ModelFeatureFlag(owners = {"hmusum"}) default int searchHandlerThreadpool() { return 2; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 3cd296c1469..7bdd2ce51a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -49,16 +49,19 @@ class SearchHandler extends ProcessingHandler<SearchChains> { private static class Threadpool extends ContainerThreadpool { + private final int threads; + Threadpool(DeployState ds, Element options) { super(ds, "search-handler", options); + threads = ds.featureFlags().searchHandlerThreadpool(); } @Override public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { builder.maxThreadExecutionTimeSeconds(190) .keepAliveTime(5.0) - .maxThreads(-2) - .minThreads(-2) + .maxThreads(-threads) + .minThreads(-threads) .queueSize(-40); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 96b0b03c832..029158056b8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -29,7 +29,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever; import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flag; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; @@ -210,6 +209,7 @@ public class ModelContextImpl implements ModelContext { private final int contentLayerMetadataFeatureLevel; private final boolean dynamicHeapSize; private final String unknownConfigDefinition; + private final int searchHandlerThreadpool; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -254,6 +254,7 @@ public class ModelContextImpl implements ModelContext { this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL); this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE); this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION); + this.searchHandlerThreadpool = flagValue(source, appId, version, Flags.SEARCH_HANDLER_THREADPOOL); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -306,6 +307,7 @@ public class ModelContextImpl implements ModelContext { @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } @Override public String unknownConfigDefinition() { return unknownConfigDefinition; } + @Override public int searchHandlerThreadpool() { return searchHandlerThreadpool; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index eb20126304e..6855e1b0559 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -28,7 +28,7 @@ public class MockBillingController implements BillingController { Map<TenantName, PlanId> plans = new HashMap<>(); Map<TenantName, PaymentInstrument> activeInstruments = new HashMap<>(); Map<TenantName, List<Bill>> committedBills = new HashMap<>(); - Map<TenantName, Bill> uncommittedBills = new HashMap<>(); + public Map<TenantName, Bill> uncommittedBills = new HashMap<>(); Map<TenantName, List<Bill.LineItem>> unusedLineItems = new HashMap<>(); Map<TenantName, CollectionMethod> collectionMethod = new HashMap<>(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java index 53d807b0139..6f056edd226 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificate.java @@ -13,9 +13,9 @@ public record EndpointCertificate(String keyName, String certName, int version, String rootRequestId, // The id of the first request made for this certificate. Should not change. Optional<String> leafRequestId, // The id of the last known request made for this certificate. Changes on refresh, may be outdated! List<String> requestedDnsSans, String issuer, Optional<Long> expiry, - Optional<Long> lastRefreshed, Optional<String> randomizedId) { + Optional<Long> lastRefreshed, Optional<String> generatedId) { - public EndpointCertificate withRandomizedId(String randomizedId) { + public EndpointCertificate withGeneratedId(String generatedId) { return new EndpointCertificate( this.keyName, this.certName, @@ -27,7 +27,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - Optional.of(randomizedId)); + Optional.of(generatedId)); } public EndpointCertificate withKeyName(String keyName) { @@ -42,7 +42,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - this.randomizedId); + this.generatedId); } public EndpointCertificate withCertName(String certName) { @@ -57,7 +57,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - this.randomizedId); + this.generatedId); } public EndpointCertificate withVersion(int version) { @@ -72,7 +72,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - this.randomizedId); + this.generatedId); } public EndpointCertificate withLastRequested(long lastRequested) { @@ -87,7 +87,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - this.randomizedId); + this.generatedId); } public EndpointCertificate withLastRefreshed(long lastRefreshed) { @@ -102,7 +102,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, Optional.of(lastRefreshed), - this.randomizedId); + this.generatedId); } public EndpointCertificate withRootRequestId(String rootRequestId) { @@ -117,7 +117,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - this.randomizedId); + this.generatedId); } public EndpointCertificate withLeafRequestId(Optional<String> leafRequestId) { @@ -132,7 +132,7 @@ public record EndpointCertificate(String keyName, String certName, int version, this.issuer, this.expiry, this.lastRefreshed, - this.randomizedId); + this.generatedId); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 173d3e1950e..44a46195989 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; @@ -28,13 +29,15 @@ public class CloudTenant extends Tenant { private final ArchiveAccess archiveAccess; private final Optional<Instant> invalidateUserSessionsBefore; private final Optional<BillingReference> billingReference; + private final PlanId planId; /** Public for the serialization layer — do not use! */ public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator, BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRoleLastMaintained, - List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference) { + List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference, + PlanId planId) { super(name, createdAt, lastLoginInfo, Optional.empty(), tenantRoleLastMaintained, cloudAccounts); this.creator = creator; this.developerKeys = developerKeys; @@ -43,6 +46,7 @@ public class CloudTenant extends Tenant { this.archiveAccess = Objects.requireNonNull(archiveAccess); this.invalidateUserSessionsBefore = invalidateUserSessionsBefore; this.billingReference = Objects.requireNonNull(billingReference); + this.planId = Objects.requireNonNull(planId); } /** Creates a tenant with the given name, provided it passes validation. */ @@ -52,7 +56,7 @@ public class CloudTenant extends Tenant { LastLoginInfo.EMPTY, Optional.ofNullable(creator).map(SimplePrincipal::of), ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty(), - Instant.EPOCH, List.of(), Optional.empty()); + Instant.EPOCH, List.of(), Optional.empty(), PlanId.from("none")); } /** The user that created the tenant */ @@ -92,6 +96,8 @@ public class CloudTenant extends Tenant { return billingReference; } + public PlanId planId() { return planId; } + @Override public Type type() { return Type.cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index ca3c66c9f72..5e4d73954ae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -121,7 +121,6 @@ import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.high; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.low; import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.normal; -import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.counting; @@ -577,17 +576,7 @@ public class ApplicationController { .orElseGet(Tags::empty); Optional<EndpointCertificate> certificate = endpointCertificates.get(instance, deployment.zoneId(), applicationPackage.truncatedPackage().deploymentSpec()); certificate.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); - BasicServicesXml services; - try { - services = applicationPackage.truncatedPackage().services(deployment, tags); - } catch (Exception e) { - // If the basic parsing done by the controller fails, we ignore the exception here so that - // complete parsing errors are propagated from the config server. Otherwise, throwing here - // will interrupt the request while it's being streamed to the config server - log.warning("Ignoring failure to parse services.xml for deployment " + deployment + - " while streaming application package: " + Exceptions.toMessageString(e)); - services = BasicServicesXml.empty; - } + BasicServicesXml services = applicationPackage.truncatedPackage().services(deployment, tags); return controller.routing().of(deployment).prepare(services, certificate, application); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 7d19acfce80..31b213e0b59 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -10,6 +10,7 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; @@ -151,12 +152,14 @@ public abstract class LockedTenant { private final ArchiveAccess archiveAccess; private final Optional<Instant> invalidateUserSessionsBefore; private final Optional<BillingReference> billingReference; + private final PlanId planId; private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator, BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore, Instant tenantRolesLastMaintained, - List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference) { + List<CloudAccountInfo> cloudAccounts, Optional<BillingReference> billingReference, + PlanId planId) { super(name, createdAt, lastLoginInfo, tenantRolesLastMaintained, cloudAccounts); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; @@ -165,15 +168,20 @@ public abstract class LockedTenant { this.archiveAccess = archiveAccess; this.invalidateUserSessionsBefore = invalidateUserSessionsBefore; this.billingReference = billingReference; + this.planId = planId; } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), tenant.tenantRolesLastMaintained(), tenant.cloudAccounts(), tenant.billingReference()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), + tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess(), tenant.invalidateUserSessionsBefore(), + tenant.tenantRolesLastMaintained(), tenant.cloudAccounts(), tenant.billingReference(), tenant.planId()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, + archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, + cloudAccounts, billingReference, planId); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -184,56 +192,84 @@ public abstract class LockedTenant { if (keys.inverse().containsKey(simplePrincipal)) throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal))); keys.put(key, simplePrincipal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference, + planId); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, + archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, + archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore,tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore,tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } public Cloud withInvalidateUserSessionsBefore(Instant invalidateUserSessionsBefore) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, + Optional.of(invalidateUserSessionsBefore), tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } @Override public LockedTenant with(Instant tenantRolesLastMaintained) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } @Override public LockedTenant withCloudAccounts(List<CloudAccountInfo> cloudAccounts) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, billingReference); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } public Cloud with(BillingReference billingReference) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, Optional.of(billingReference)); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + Optional.of(billingReference), planId); + } + + public Cloud withPlanId(PlanId planId) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, + invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccounts, + billingReference, planId); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 90c4a506f10..b763af1af9d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -51,7 +51,6 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -135,7 +134,7 @@ public class RoutingController { } // Add zone-scoped endpoints - Map<EndpointId, GeneratedEndpointList> generatedForDeclaredEndpoints = new HashMap<>(); + Map<EndpointId, List<GeneratedEndpoint>> generatedForDeclaredEndpoints = new HashMap<>(); Set<ClusterSpec.Id> clustersWithToken = new HashSet<>(); boolean generatedEndpointsEnabled = generatedEndpointsEnabled(deployment.applicationId()); RoutingPolicyList applicationPolicies = policies().read(TenantAndApplicationId.from(deployment.applicationId())); @@ -149,9 +148,10 @@ public class RoutingController { Optional<RoutingPolicy> clusterPolicy = deploymentPolicies.cluster(clusterId).first(); List<GeneratedEndpoint> generatedForCluster = clusterPolicy.map(policy -> policy.generatedEndpoints().cluster().asList()) .orElseGet(List::of); - // Generate endpoints if cluster does not have any - if (generatedForCluster.isEmpty()) { - generatedForCluster = generateEndpoints(tokenSupported, certificate, Optional.empty()); + // Generate endpoint for each auth method, if not present + generatedForCluster = generateEndpoints(AuthMethod.mtls, certificate, Optional.empty(), generatedForCluster); + if (tokenSupported) { + generatedForCluster = generateEndpoints(AuthMethod.token, certificate, Optional.empty(), generatedForCluster); } GeneratedEndpointList generatedEndpoints = generatedEndpointsEnabled ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedEndpoints).scope(Scope.zone)); @@ -162,18 +162,34 @@ public class RoutingController { ClusterSpec.Id clusterId = ClusterSpec.Id.from(container.id()); applicationPolicies.cluster(clusterId).asList().stream() .flatMap(policy -> policy.generatedEndpoints().declared().asList().stream()) - .forEach(ge -> generatedForDeclaredEndpoints.computeIfAbsent(ge.endpoint().get(), (k) -> GeneratedEndpointList.of(ge))); + .forEach(ge -> { + List<GeneratedEndpoint> generated = generatedForDeclaredEndpoints.computeIfAbsent(ge.endpoint().get(), (k) -> new ArrayList<>()); + if (!generated.contains(ge)) { + generated.add(ge); + } + }); } // Generate endpoints if declared endpoint does not have any Stream.concat(spec.endpoints().stream(), spec.instances().stream().flatMap(i -> i.endpoints().stream())) .forEach(endpoint -> { EndpointId endpointId = EndpointId.of(endpoint.endpointId()); - generatedForDeclaredEndpoints.computeIfAbsent(endpointId, (k) -> { + generatedForDeclaredEndpoints.compute(endpointId, (k, old) -> { + if (old == null) { + old = List.of(); + } + List<GeneratedEndpoint> generatedEndpoints = generateEndpoints(AuthMethod.mtls, certificate, Optional.of(endpointId), old); boolean tokenSupported = clustersWithToken.contains(ClusterSpec.Id.from(endpoint.containerId())); - return generatedEndpointsEnabled ? GeneratedEndpointList.copyOf(generateEndpoints(tokenSupported, certificate, Optional.of(endpointId))) : null; + if (tokenSupported){ + generatedEndpoints = generateEndpoints(AuthMethod.token, certificate, Optional.of(endpointId), generatedEndpoints); + } + return generatedEndpoints; }); }); - Map<EndpointId, GeneratedEndpointList> generatedEndpoints = generatedEndpointsEnabled ? generatedForDeclaredEndpoints : Map.of(); + Map<EndpointId, GeneratedEndpointList> generatedEndpoints = generatedEndpointsEnabled + ? generatedForDeclaredEndpoints.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, kv -> GeneratedEndpointList.copyOf(kv.getValue()))) + : Map.of(); endpoints = endpoints.and(declaredEndpointsOf(application.get().id(), spec, generatedEndpoints).targets(deployment)); PreparedEndpoints prepared = new PreparedEndpoints(deployment, endpoints, @@ -186,12 +202,6 @@ public class RoutingController { return prepared; } - private List<GeneratedEndpoint> generateEndpoints(boolean tokenSupported, Optional<EndpointCertificate> certificate, Optional<EndpointId> endpoint) { - return certificate.flatMap(EndpointCertificate::randomizedId) - .map(id -> generateEndpoints(id, tokenSupported, endpoint)) - .orElseGet(List::of); - } - // -------------- Implicit endpoints (scopes 'zone' and 'weighted') -------------- /** Returns the zone- and region-scoped endpoints of given deployment */ @@ -480,19 +490,22 @@ public class RoutingController { } } - /** Generate endpoints for all authentication methods, using given application part */ - private List<GeneratedEndpoint> generateEndpoints(String applicationPart, boolean token, Optional<EndpointId> endpoint) { - return Arrays.stream(AuthMethod.values()) - .filter(method -> switch (method) { - case token -> token; - case mtls -> true; - case none -> false; - }) - .map(method -> new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), - applicationPart, - method, - endpoint)) - .toList(); + /** Returns generated endpoints. A new endpoint is generated if no matching endpoint already exists */ + private List<GeneratedEndpoint> generateEndpoints(AuthMethod authMethod, Optional<EndpointCertificate> certificate, + Optional<EndpointId> declaredEndpoint, + List<GeneratedEndpoint> current) { + if (current.stream().anyMatch(e -> e.authMethod() == authMethod && e.endpoint().equals(declaredEndpoint))) { + return current; + } + Optional<String> applicationPart = certificate.flatMap(EndpointCertificate::generatedId); + if (applicationPart.isPresent()) { + current = new ArrayList<>(current); + current.add(new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), + applicationPart.get(), + authMethod, + declaredEndpoint)); + } + return current; } /** Generate the cluster part of a {@link GeneratedEndpoint} for use in a {@link Endpoint.Scope#weighted} endpoint */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java index 8f3a02528fe..6a634cb8f53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java @@ -9,7 +9,6 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.TreeSet; /** * A partially parsed variant of services.xml, for use by the {@link com.yahoo.vespa.hosted.controller.Controller}. @@ -40,7 +39,9 @@ public record BasicServicesXml(List<Container> containers) { for (var childNode : XML.getChildren(root)) { if (childNode.getTagName().equals(CONTAINER_TAG)) { String id = childNode.getAttribute("id"); - if (id.isEmpty()) throw new IllegalArgumentException(CONTAINER_TAG + " tag requires 'id' attribute"); + if (id.isEmpty()) { + id = CONTAINER_TAG; // ID defaults to tag name when unset. See ConfigModelBuilder::getIdString + } List<Container.AuthMethod> methods = new ArrayList<>(); List<TokenId> tokens = new ArrayList<>(); parseAuthMethods(childNode, methods, tokens); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index b5e012253c7..ec2ef4b7ff8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -119,11 +119,11 @@ public class EndpointCertificates { TenantAndApplicationId application = TenantAndApplicationId.from(instance.id()); Optional<AssignedCertificate> perInstanceAssignedCertificate = curator.readAssignedCertificate(application, Optional.of(instance.name())); - if (perInstanceAssignedCertificate.isPresent() && perInstanceAssignedCertificate.get().certificate().randomizedId().isPresent()) { + if (perInstanceAssignedCertificate.isPresent() && perInstanceAssignedCertificate.get().certificate().generatedId().isPresent()) { return updateLastRequested(perInstanceAssignedCertificate.get()).certificate(); } else if (! zone.environment().isManuallyDeployed()) { Optional<AssignedCertificate> perApplicationAssignedCertificate = curator.readAssignedCertificate(application, Optional.empty()); - if (perApplicationAssignedCertificate.isPresent() && perApplicationAssignedCertificate.get().certificate().randomizedId().isPresent()) { + if (perApplicationAssignedCertificate.isPresent() && perApplicationAssignedCertificate.get().certificate().generatedId().isPresent()) { return updateLastRequested(perApplicationAssignedCertificate.get()).certificate(); } } @@ -181,7 +181,7 @@ public class EndpointCertificates { boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); - var requiredSansForZone = legacyNames || currentCertificate.get().randomizedId().isEmpty() ? + var requiredSansForZone = legacyNames || currentCertificate.get().generatedId().isEmpty() ? controller.routing().certificateDnsNames(deployment, deploymentSpec) : List.<String>of(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java index 3a8580b7eb5..1566949664b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/UnassignedCertificate.java @@ -14,13 +14,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe public record UnassignedCertificate(EndpointCertificate certificate, UnassignedCertificate.State state) { public UnassignedCertificate { - if (certificate.randomizedId().isEmpty()) { - throw new IllegalArgumentException("randomizedId must be set for a pooled certificate"); + if (certificate.generatedId().isEmpty()) { + throw new IllegalArgumentException("generatedId must be set for a pooled certificate"); } } public String id() { - return certificate.randomizedId().get(); + return certificate.generatedId().get(); } public UnassignedCertificate withState(State state) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index ed383175cc3..a07b6e14625 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -106,7 +106,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { curator.readAssignedCertificates().stream() .map(AssignedCertificate::certificate) - .map(EndpointCertificate::randomizedId) + .map(EndpointCertificate::generatedId) .forEach(id -> id.ifPresent(existingNames::add)); String id = generateRandomId(); @@ -122,7 +122,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { Optional.empty(), endpointCertificateAlgo.value(), useAlternateCertProvider.value()) - .withRandomizedId(id); + .withGeneratedId(id); UnassignedCertificate certificate = new UnassignedCertificate(f, UnassignedCertificate.State.requested); curator.writeUnassignedCertificate(certificate); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 805bf3d7ada..f4936dcfa8b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; @@ -280,7 +279,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { */ assignedCertificates.stream() .filter(c -> c.instance().isPresent()) - .filter(c -> c.certificate().randomizedId().isEmpty()) + .filter(c -> c.certificate().generatedId().isEmpty()) .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion .limit(assignRandomizedIdRate.value()) .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); @@ -300,7 +299,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { log.log(Level.INFO, "Assigned certificate missing for " + tenantAndApplicationId.instance(instanceName).toFullString() + " when assigning randomized id"); } // Verify that the assigned certificate still does not have randomized id assigned - if (assignedCertificate.get().certificate().randomizedId().isPresent()) return; + if (assignedCertificate.get().certificate().generatedId().isPresent()) return; controller().applications().lockApplicationOrThrow(tenantAndApplicationId, application -> { DeploymentSpec deploymentSpec = application.get().deploymentSpec(); @@ -320,7 +319,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { EndpointCertificate withRandomNames = requestRandomNames( tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), - applicationLevelAssignedCertificate.get().certificate().randomizedId() + applicationLevelAssignedCertificate.get().certificate().generatedId() .orElseThrow(() -> new IllegalArgumentException("Application certificate already assigned to " + tenantAndApplicationId.toString() + ", but random id is missing")), Optional.of(instanceLevelAssignedCertificate.certificate())); AssignedCertificate assignedCertWithRandomNames = instanceLevelAssignedCertificate.with(withRandomNames); @@ -365,12 +364,12 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { previousRequest, endpointCertificateAlgo.value(), useAlternateCertProvider.value()) - .withRandomizedId(randomId); + .withGeneratedId(randomId); } private String generateRandomId() { List<String> unassignedIds = curator.readUnassignedCertificates().stream().map(UnassignedCertificate::id).toList(); - List<String> assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::randomizedId).filter(Optional::isPresent).map(Optional::get).toList(); + List<String> assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::generatedId).filter(Optional::isPresent).map(Optional::get).toList(); Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); String randomId; do { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java index fae9ea1e0e3..2ff4f1fb194 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateSerializer.java @@ -35,7 +35,7 @@ public class EndpointCertificateSerializer { private final static String issuerField = "issuer"; private final static String expiryField = "expiry"; private final static String lastRefreshedField = "lastRefreshed"; - private final static String randomizedIdField = "randomizedId"; + private final static String generatedIdField = "randomizedId"; public static Slime toSlime(EndpointCertificate cert) { Slime slime = new Slime(); @@ -56,7 +56,7 @@ public class EndpointCertificateSerializer { object.setString(issuerField, cert.issuer()); cert.expiry().ifPresent(expiry -> object.setLong(expiryField, expiry)); cert.lastRefreshed().ifPresent(refreshTime -> object.setLong(lastRefreshedField, refreshTime)); - cert.randomizedId().ifPresent(randomizedId -> object.setString(randomizedIdField, randomizedId)); + cert.generatedId().ifPresent(id -> object.setString(generatedIdField, id)); } public static EndpointCertificate fromSlime(Inspector inspector) { @@ -79,8 +79,8 @@ public class EndpointCertificateSerializer { inspector.field(lastRefreshedField).valid() ? Optional.of(inspector.field(lastRefreshedField).asLong()) : Optional.empty(), - inspector.field(randomizedIdField).valid() ? - Optional.of(inspector.field(randomizedIdField).asString()) : + inspector.field(generatedIdField).valid() ? + Optional.of(inspector.field(generatedIdField).asString()) : Optional.empty()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 760fb9b0366..166418a54f7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -15,6 +15,7 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; @@ -88,6 +89,7 @@ public class TenantSerializer { private static final String invalidateUserSessionsBeforeField = "invalidateUserSessionsBefore"; private static final String tenantRolesLastMaintainedField = "tenantRolesLastMaintained"; private static final String billingReferenceField = "billingReference"; + private static final String planIdField = "planId"; private static final String cloudAccountsField = "cloudAccounts"; private static final String accountField = "account"; private static final String templateVersionField = "templateVersion"; @@ -137,6 +139,7 @@ public class TenantSerializer { toSlime(tenant.archiveAccess(), root); tenant.billingReference().ifPresent(b -> toSlime(b, root)); tenant.invalidateUserSessionsBefore().ifPresent(instant -> root.setLong(invalidateUserSessionsBeforeField, instant.toEpochMilli())); + root.setString(planIdField, tenant.planId().value()); } private void toSlime(ArchiveAccess archiveAccess, Cursor root) { @@ -215,7 +218,10 @@ public class TenantSerializer { Instant tenantRolesLastMaintained = SlimeUtils.instant(tenantObject.field(tenantRolesLastMaintainedField)); List<CloudAccountInfo> cloudAccountInfos = cloudAccountsFromSlime(tenantObject.field(cloudAccountsField)); Optional<BillingReference> billingReference = billingReferenceFrom(tenantObject.field(billingReferenceField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, cloudAccountInfos, billingReference); + PlanId planId = planId(tenantObject.field(planIdField)); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, + archiveAccess, invalidateUserSessionsBefore, tenantRolesLastMaintained, + cloudAccountInfos, billingReference, planId); } private DeletedTenant deletedTenantFrom(Inspector tenantObject) { @@ -250,6 +256,7 @@ public class TenantSerializer { .withAWSRole(awsArchiveAccessRole) .withGCPMember(gcpArchiveAccessMember); } + TenantInfo tenantInfoFromSlime(Inspector infoObject) { if (!infoObject.valid()) return TenantInfo.empty(); @@ -375,6 +382,12 @@ public class TenantSerializer { SlimeUtils.instant(object.field("updated")))); } + private PlanId planId(Inspector object) { + if (! object.valid()) return PlanId.from("none"); + + return PlanId.from(object.asString()); + } + private TenantContacts tenantContactsFrom(Inspector object) { List<TenantContacts.Contact> contacts = SlimeUtils.entriesStream(object) .map(this::readContact) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 05d88a12251..cc031fb27c7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -31,7 +31,9 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.math.BigDecimal; import java.time.Clock; +import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Comparator; @@ -82,6 +84,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler */ .addRoute(RestApi.route("/billing/v2/accountant") .get(self::accountant)) + .addRoute(RestApi.route("/billing/v2/accountant/preview") + .get(self::accountantPreview)) .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) @@ -202,21 +206,39 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler // --------- ACCOUNTANT API ---------- private Slime accountant(RestApi.RequestContext requestContext) { - var untilAt = untilParameter(requestContext); - var usagePerTenant = billing.createUncommittedBills(untilAt); - var response = new Slime(); var tenantsResponse = response.setObject().setArray("tenants"); tenants.asList().stream().sorted(Comparator.comparing(Tenant::name)).forEach(tenant -> { - var usage = Optional.ofNullable(usagePerTenant.get(tenant.name())); var tenantResponse = tenantsResponse.addObject(); tenantResponse.setString("tenant", tenant.name().value()); toSlime(tenantResponse.setObject("plan"), planFor(tenant.name())); toSlime(tenantResponse.setObject("quota"), billing.getQuota(tenant.name())); tenantResponse.setString("collection", billing.getCollectionMethod(tenant.name()).name()); - tenantResponse.setString("lastBill", usage.map(Bill::getStartDate).map(DateTimeFormatter.ISO_DATE::format).orElse(null)); - tenantResponse.setString("unbilled", usage.map(Bill::sum).map(BigDecimal::toPlainString).orElse("0.00")); + tenantResponse.setString("lastBill", LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC).format(DateTimeFormatter.ISO_DATE)); + tenantResponse.setString("unbilled", "0.00"); + }); + + return response; + } + + private Slime accountantPreview(RestApi.RequestContext requestContext) { + var untilAt = untilParameter(requestContext); + var usagePerTenant = billing.createUncommittedBills(untilAt); + + var response = new Slime(); + var tenantsResponse = response.setObject().setArray("tenants"); + + usagePerTenant.entrySet().stream().sorted(Comparator.comparing(x -> x.getValue().sum())).forEachOrdered(x -> { + var tenant = x.getKey(); + var usage = x.getValue(); + var tenantResponse = tenantsResponse.addObject(); + tenantResponse.setString("tenant", tenant.value()); + toSlime(tenantResponse.setObject("plan"), planFor(tenant)); + toSlime(tenantResponse.setObject("quota"), billing.getQuota(tenant)); + tenantResponse.setString("collection", billing.getCollectionMethod(tenant).name()); + tenantResponse.setString("lastBill", usage.getStartDate().format(DateTimeFormatter.ISO_DATE)); + tenantResponse.setString("unbilled", usage.sum().toPlainString()); }); return response; @@ -273,7 +295,10 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var exportMethod = slime.get().field("method").asString(); var result = billing.exportBill(bill, exportMethod, cloudTenant); - return new MessageResponse("Bill has been exported: " + result); + + var responseSlime = new Slime(); + responseSlime.setObject().setString("invoiceId", result); + return new SlimeJsonResponse(responseSlime); } // --------- INVOICE RENDERING ---------- diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java index c28185466b0..f7bd1e06e68 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java @@ -18,11 +18,13 @@ class BasicServicesXmlTest { public void parse() { assertServices(new BasicServicesXml(List.of()), "<services/>"); assertServices(new BasicServicesXml(List.of(new Container("foo", List.of(Container.AuthMethod.mtls), List.of()), - new Container("bar", List.of(Container.AuthMethod.mtls), List.of()))), + new Container("bar", List.of(Container.AuthMethod.mtls), List.of()), + new Container("container", List.of(Container.AuthMethod.mtls), List.of()))), """ <services> <container id="foo"/> <container id="bar"/> + <container/> </services> """); assertServices(new BasicServicesXml(List.of( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index a6d3b435dcb..2bc11adddf7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -296,7 +295,7 @@ public class EndpointCertificatesTest { // Initial certificate is requested directly from provider Optional<EndpointCertificate> certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); assertTrue(certFromProvider.isPresent()); - assertFalse(certFromProvider.get().randomizedId().isPresent()); + assertFalse(certFromProvider.get().generatedId().isPresent()); // Pooled certificates become available tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); @@ -315,8 +314,8 @@ public class EndpointCertificatesTest { String certId = "pool-cert-1"; addCertificateToPool(certId, UnassignedCertificate.State.ready); Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertEquals(certId, cert.get().randomizedId().get()); - assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().randomizedId().get(), "Certificate is assigned at application-level"); + assertEquals(certId, cert.get().generatedId().get()); + assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().generatedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } @@ -326,8 +325,8 @@ public class EndpointCertificatesTest { addCertificateToPool(certId, UnassignedCertificate.State.ready); ZoneId devZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); Optional<EndpointCertificate> cert = endpointCertificates.get(instance, devZone, DeploymentSpec.empty); - assertEquals(certId, cert.get().randomizedId().get()); - assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().randomizedId().get(), "Certificate is assigned at instance-level"); + assertEquals(certId, cert.get().generatedId().get()); + assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().generatedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } @@ -338,7 +337,7 @@ public class EndpointCertificatesTest { // Initial certificate is requested directly from provider Optional<EndpointCertificate> certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); assertTrue(certFromProvider.isPresent()); - assertFalse(certFromProvider.get().randomizedId().isPresent()); + assertFalse(certFromProvider.get().generatedId().isPresent()); // Simulate endpoint certificate maintainer to assign random id TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(instance.id()); @@ -346,7 +345,7 @@ public class EndpointCertificatesTest { Optional<AssignedCertificate> assignedCertificate = tester.controller().curator().readAssignedCertificate(tenantAndApplicationId, instanceName); assertTrue(assignedCertificate.isPresent()); String assignedRandomId = "randomid"; - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withRandomizedId(assignedRandomId)); + AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withGeneratedId(assignedRandomId)); tester.controller().curator().writeAssignedCertificate(updated); // Pooled certificates become available @@ -358,12 +357,12 @@ public class EndpointCertificatesTest { // Request cert for app Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertEquals(assignedRandomId, cert.get().randomizedId().get()); + assertEquals(assignedRandomId, cert.get().generatedId().get()); // Pooled cert remains unassigned List<String> unassignedCertificateIds = tester.curator().readUnassignedCertificates().stream() .map(UnassignedCertificate::certificate) - .map(EndpointCertificate::randomizedId) + .map(EndpointCertificate::generatedId) .map(Optional::get) .toList(); assertEquals(List.of(certId), unassignedCertificateIds); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 647c809231e..2f996bac897 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -41,7 +41,6 @@ import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.perfUsEast3; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; @@ -138,7 +137,7 @@ public class EndpointCertificateMaintainerTest { tester.clock().advance(Duration.ofDays(3)); secretStore.setSecret(assignedCertificate.certificate().keyName(), "foo", 1); secretStore.setSecret(assignedCertificate.certificate().certName(), "bar", 1); - tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate("preprovisioned." + assignedCertificate.certificate().randomizedId().get(), assignedCertificate.certificate().requestedDnsSans(), Optional.of(assignedCertificate.certificate()), "rsa_2048", false); + tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate("preprovisioned." + assignedCertificate.certificate().generatedId().get(), assignedCertificate.certificate().requestedDnsSans(), Optional.of(assignedCertificate.certificate()), "rsa_2048", false); // We should now pick up the new key and cert version + uuid, but not force trigger deployment yet assertEquals(0.0, maintainer.maintain(), 0.0000001); @@ -206,7 +205,7 @@ public class EndpointCertificateMaintainerTest { assertTrue(applicationCertificate.isPresent()); Optional<AssignedCertificate> instanceCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.of(app.instance())); assertTrue(instanceCertificate.isPresent()); - assertEquals(instanceCertificate.get().certificate().randomizedId(), applicationCertificate.get().certificate().randomizedId()); + assertEquals(instanceCertificate.get().certificate().generatedId(), applicationCertificate.get().certificate().generatedId()); // Verify the 3 wildcard random names are same in all certs List<String> appWildcardSans = applicationCertificate.get().certificate().requestedDnsSans(); @@ -226,13 +225,13 @@ public class EndpointCertificateMaintainerTest { assertEquals(1, tester.curator().readAssignedCertificates().size()); maintainer.maintain(); - String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().randomizedId().get(); + String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().generatedId().get(); deployToAssignCert(deploymentTester, instance2, List.of(productionUsWest1), Optional.of("instance1,instance2")); maintainer.maintain(); assertEquals(3, tester.curator().readAssignedCertificates().size()); - assertEquals(randomId, tester.curator().readAssignedCertificate(instance1).get().certificate().randomizedId().get()); + assertEquals(randomId, tester.curator().readAssignedCertificate(instance1).get().certificate().generatedId().get()); } @Test @@ -247,7 +246,7 @@ public class EndpointCertificateMaintainerTest { // Verify certificate is assigned random id and 3 new names Optional<AssignedCertificate> assignedCertificate = tester.curator().readAssignedCertificate(devApp); - assertTrue(assignedCertificate.get().certificate().randomizedId().isPresent()); + assertTrue(assignedCertificate.get().certificate().generatedId().isPresent()); List<String> newRequestedSans = assignedCertificate.get().certificate().requestedDnsSans(); List<String> randomizedNames = newRequestedSans.stream().filter(san -> !originalRequestedSans.contains(san)).toList(); assertEquals(3, randomizedNames.size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 228a61cebc6..373d4661f17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -69,7 +70,8 @@ public class NotificationsDbTest { Optional.empty(), Instant.EPOCH, List.of(), - Optional.empty()); + Optional.empty(), + PlanId.from("none")); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index 15524e2748c..b264ec40f7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -47,7 +48,8 @@ public class NotifierTest { Optional.empty(), Instant.EPOCH, List.of(), - Optional.empty()); + Optional.empty(), + PlanId.from("none")); MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 4369675ba3e..d13ec5e85d2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -12,6 +12,7 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; @@ -114,12 +115,14 @@ public class TenantSerializerTest { Optional.empty(), Instant.EPOCH, List.of(), - Optional.empty()); + Optional.empty(), + PlanId.from("none")); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.creator(), serialized.creator()); assertEquals(tenant.developerKeys(), serialized.developerKeys()); assertEquals(tenant.createdAt(), serialized.createdAt()); + assertEquals("none", serialized.planId().value()); } @Test @@ -139,7 +142,8 @@ public class TenantSerializerTest { Optional.of(Instant.ofEpochMilli(1234567)), Instant.EPOCH, List.of(), - Optional.empty()); + Optional.empty(), + PlanId.from("none")); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); assertEquals(tenant.tenantSecretStores(), serialized.tenantSecretStores()); @@ -193,7 +197,8 @@ public class TenantSerializerTest { Instant.EPOCH, List.of(new CloudAccountInfo(CloudAccount.from("aws:123456789012"), Version.fromString("1.2.3")), new CloudAccountInfo(CloudAccount.from("gcp:my-project"), Version.fromString("3.2.1"))), - Optional.empty()); + Optional.empty(), + PlanId.from("none")); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(serialized.archiveAccess().awsRole().get(), "arn:aws:iam::123456789012:role/my-role"); assertEquals(serialized.archiveAccess().gcpMember().get(), "user:foo@example.com"); @@ -253,6 +258,30 @@ public class TenantSerializerTest { } @Test + void cloud_tenant_with_plan_id() { + CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.ofEpochMilli(1234L), + lastLoginInfo(123L, 456L, null), + Optional.of(new SimplePrincipal("foobar-user")), + ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), + otherPublicKey, new SimplePrincipal("jane")), + TenantInfo.empty(), + List.of(), + new ArchiveAccess(), + Optional.empty(), + Instant.EPOCH, + List.of(), + Optional.empty(), + PlanId.from("pay-as-you-go")); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + assertEquals(tenant.name(), serialized.name()); + assertEquals(tenant.creator(), serialized.creator()); + assertEquals(tenant.developerKeys(), serialized.developerKeys()); + assertEquals(tenant.createdAt(), serialized.createdAt()); + assertEquals(tenant.planId(), serialized.planId()); + } + + @Test void deleted_tenant() { DeletedTenant tenant = new DeletedTenant( TenantName.from("tenant1"), Instant.ofEpochMilli(1234L), Instant.ofEpochMilli(2345L)); @@ -291,7 +320,8 @@ public class TenantSerializerTest { Optional.empty(), Instant.EPOCH, List.of(), - Optional.of(reference)); + Optional.of(reference), + PlanId.from("none")); var slime = serializer.toSlime(tenant); var deserialized = serializer.tenantFrom(slime); assertEquals(tenant, deserialized); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 43271277ce9..94898b706c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -116,7 +116,16 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var accountantRequest = request("/billing/v2/accountant").roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, """ - {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"quota":{"budget":-1.0},"collection":"AUTO","lastBill":null,"unbilled":"0.00"}]}"""); + {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"quota":{"budget":-1.0},"collection":"AUTO","lastBill":"1970-01-01","unbilled":"0.00"}]}"""); + } + + @Test + void require_accountant_preview() { + var accountantRequest = request("/billing/v2/accountant/preview").roles(Role.hostedAccountant()); + billingController.uncommittedBills.put(tenant, BillingApiHandlerTest.createBill()); + + tester.assertResponse(accountantRequest, """ + {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"quota":{"budget":-1.0},"collection":"AUTO","lastBill":"2020-05-23","unbilled":"123.00"}]}"""); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 001e02e1b16..8b0a4287dc3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -11,6 +11,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; @@ -120,7 +121,8 @@ public class SignatureFilterTest { Optional.empty(), Instant.EPOCH, List.of(), - Optional.empty())); + Optional.empty(), + PlanId.from("none"))); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), Set.of(Role.reader(id.tenant()), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 22523103208..3405009714d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1206,6 +1206,70 @@ public class RoutingPoliciesTest { } @Test + public void generated_endpoints_enable_token() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + + // Deploy application without token + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .container("c0", AuthMethod.mtls) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), zone1); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud", + "ebd395b6.cafed00d.z.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"), + tester.recordNames()); + + // Re-deploy with token enabled + applicationPackage = applicationPackageBuilder().region(zone1.region()) + .container("c0", AuthMethod.mtls, AuthMethod.token) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + // Additional zone- and global-scoped endpoints are added (token) + assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud", + "b7e79800.cafed00d.z.vespa-app.cloud", + "c60d3149.cafed00d.g.vespa-app.cloud", + "ebd395b6.cafed00d.z.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"), + tester.recordNames()); + + // Add new endpoint is generated for an additional global endpoint + applicationPackage = applicationPackageBuilder().region(zone1.region()) + .container("c0", AuthMethod.mtls, AuthMethod.token) + .endpoint("foo", "c0") + .endpoint("bar", "c0") + .build(); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + List<String> expectedRecords = List.of("a9c8c045.cafed00d.g.vespa-app.cloud", + "aa7591aa.cafed00d.g.vespa-app.cloud", + "b7e79800.cafed00d.z.vespa-app.cloud", + "c60d3149.cafed00d.g.vespa-app.cloud", + "d467800f.cafed00d.g.vespa-app.cloud", + "ebd395b6.cafed00d.z.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"); + assertEquals(expectedRecords, tester.recordNames()); + + // No change on redeployment + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + assertEquals(expectedRecords, tester.recordNames()); + } + + @Test public void generated_endpoints_only() { var tester = new RoutingPoliciesTester(SystemName.Public); var context = tester.newDeploymentContext("tenant1", "app1", "default"); @@ -1216,6 +1280,7 @@ public class RoutingPoliciesTest { // Deploy application var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .container("c0", AuthMethod.mtls) .endpoint("foo", "c0") @@ -1232,6 +1297,23 @@ public class RoutingPoliciesTest { "ebd395b6.cafed00d.z.vespa-app.cloud", "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"), tester.recordNames()); + + // Another zone is added to global endpoint + applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .container("c0", AuthMethod.mtls) + .endpoint("foo", "c0") + .build(); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); + tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); + assertEquals(List.of("a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud", + "a9c8c045.cafed00d.g.vespa-app.cloud", + "cbff1506.cafed00d.z.vespa-app.cloud", + "ebd395b6.cafed00d.z.vespa-app.cloud", + "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud"), + tester.recordNames()); } @Test diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index e902868c10a..e8f3d183abd 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -95,7 +95,7 @@ <hdrhistogram.vespa.version>2.1.12</hdrhistogram.vespa.version> <huggingface.vespa.version>0.24.0</huggingface.vespa.version> <icu4j.vespa.version>73.2</icu4j.vespa.version> - <java-jjwt.vespa.version>0.12.0</java-jjwt.vespa.version> + <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> <java-jwt.vespa.version>4.4.0</java-jwt.vespa.version> <jaxb.runtime.vespa.version>4.0.3</jaxb.runtime.vespa.version> <jetty.vespa.version>11.0.16</jetty.vespa.version> @@ -121,7 +121,7 @@ <org.json.vespa.version>20230618</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> - <protobuf.vespa.version>3.24.3</protobuf.vespa.version> + <protobuf.vespa.version>3.24.4</protobuf.vespa.version> <questdb.vespa.version>7.3.2</questdb.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> @@ -137,12 +137,12 @@ <junit.platform.vespa.tenant.version>1.8.1</junit.platform.vespa.tenant.version> <!-- Maven plugins --> - <clover-maven-plugin.vespa.version>4.4.1</clover-maven-plugin.vespa.version> + <clover-maven-plugin.vespa.version>4.5.0</clover-maven-plugin.vespa.version> <maven-antrun-plugin.vespa.version>3.1.0</maven-antrun-plugin.vespa.version> <maven-assembly-plugin.vespa.version>3.6.0</maven-assembly-plugin.vespa.version> <maven-bundle-plugin.vespa.version>5.1.9</maven-bundle-plugin.vespa.version> <maven-compiler-plugin.vespa.version>3.11.0</maven-compiler-plugin.vespa.version> - <maven-core.vespa.version>3.9.4</maven-core.vespa.version> + <maven-core.vespa.version>3.9.5</maven-core.vespa.version> <maven-dependency-plugin.vespa.version>3.6.0</maven-dependency-plugin.vespa.version> <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> @@ -154,6 +154,7 @@ <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> <maven-plugin-tools.vespa.version>3.9.0</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> + <maven-resolver.vespa.version>1.9.16</maven-resolver.vespa.version> <maven-shade-plugin.vespa.version>3.5.1</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.0</maven-source-plugin.vespa.version> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 0d187514e53..27c9e9ee7da 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -422,6 +422,13 @@ public class Flags { "Takes effect on redeployment through controller", INSTANCE_ID, APPLICATION_ID, TENANT_ID); + public static final UnboundIntFlag SEARCH_HANDLER_THREADPOOL = defineIntFlag( + "search-handler-threadpool", 2, + List.of("bjorncs", "baldersheim"), "2023-10-01", "2024-01-01", + "Adjust search handler threadpool size", + "Takes effect at redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 06f2f34964b..3ad61c7bc0e 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -26,11 +26,11 @@ org.apache.maven.enforcer:enforcer-api:${maven-enforcer-plugin.vespa.version} org.apache.maven.enforcer:enforcer-rules:${maven-enforcer-plugin.vespa.version} org.apache.maven.plugin-tools:maven-plugin-annotations:${maven-plugin-tools.vespa.version} org.apache.maven.plugins:maven-shade-plugin:${maven-shade-plugin.vespa.version} -org.apache.maven.resolver:maven-resolver-api:1.9.14 -org.apache.maven.resolver:maven-resolver-impl:1.9.14 -org.apache.maven.resolver:maven-resolver-named-locks:1.9.14 -org.apache.maven.resolver:maven-resolver-spi:1.9.14 -org.apache.maven.resolver:maven-resolver-util:1.9.14 +org.apache.maven.resolver:maven-resolver-api:${maven-resolver.vespa.version} +org.apache.maven.resolver:maven-resolver-impl:${maven-resolver.vespa.version} +org.apache.maven.resolver:maven-resolver-named-locks:${maven-resolver.vespa.version} +org.apache.maven.resolver:maven-resolver-spi:${maven-resolver.vespa.version} +org.apache.maven.resolver:maven-resolver-util:${maven-resolver.vespa.version} org.apache.maven.shared:maven-dependency-tree:3.2.1 org.apache.maven.shared:maven-shared-utils:3.3.4 org.apache.maven:maven-archiver:${maven-archiver.vespa.version} diff --git a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java index 0b0ebde2f1d..fdc24d7e697 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java @@ -39,6 +39,7 @@ public enum ControllerMetrics implements VespaMetrics { COREDUMP_PROCESSED("coredump.processed", Unit.FAILURE,"Controller: Core dumps processed"), AUTH0_EXCEPTIONS("auth0.exceptions", Unit.FAILURE, "Controller: Auth0 exceptions"), CERTIFICATE_POOL_AVAILABLE("certificate_pool_available", Unit.FRACTION, "Available certificates in the pool, fraction of configured size"), + BILLING_EXCEPTIONS("billing.exceptions", Unit.FAILURE, "Controller: Billing related exceptions"), // Metrics per API, metrics names generated in ControllerMaintainer/MetricsReporter OPERATION_APPLICATION("operation.application", Unit.REQUEST, "Controller: Requests for /application API"), diff --git a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index e1f694d1dc7..f763c0a27e1 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -174,6 +174,7 @@ public class InfrastructureMetricSet { addMetric(metrics, ControllerMetrics.COREDUMP_PROCESSED.count()); addMetric(metrics, ControllerMetrics.AUTH0_EXCEPTIONS.count()); addMetric(metrics, ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.max()); + addMetric(metrics, ControllerMetrics.BILLING_EXCEPTIONS.count()); addMetric(metrics, ControllerMetrics.METERING_AGE_SECONDS.min()); addMetric(metrics, ControllerMetrics.METERING_LAST_REPORTED.max()); diff --git a/parent/pom.xml b/parent/pom.xml index 6b93a67803e..244d6b07710 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.7.1</version> + <version>5.8.0</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -891,6 +891,31 @@ <version>${maven-wagon.vespa.version}</version> </dependency> <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-api</artifactId> + <version>${maven-resolver.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-impl</artifactId> + <version>${maven-resolver.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-named-locks</artifactId> + <version>${maven-resolver.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-spi</artifactId> + <version>${maven-resolver.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.maven.resolver</groupId> + <artifactId>maven-resolver-util</artifactId> + <version>${maven-resolver.vespa.version}</version> + </dependency> + <dependency> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> <version>${opennlp.vespa.version}</version> diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 71dfed86fdb..6d0c025038a 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -6,14 +6,12 @@ #include "randreaders.h" #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/arrayqueue.hpp> -#include <vespa/vespalib/util/array.hpp> #include <vespa/fastos/file.h> #include <filesystem> #include <future> @@ -117,33 +115,14 @@ FileChunk::addNumBuckets(size_t numBucketsInChunk) } } -class TmpChunkMeta : public ChunkMeta, - public std::vector<LidMeta> -{ -public: - void fill(vespalib::nbostream & is) { - resize(getNumEntries()); - for (LidMeta & lm : *this) { - lm.deserialize(is); - } - } -}; - -using TmpChunkMetaV = std::vector<TmpChunkMeta>; - -namespace { - void -verifyOrAssert(const TmpChunkMetaV & v) -{ - for (auto prev(v.begin()), it(prev); it != v.end(); ++it) { - assert(prev->getLastSerial() <= it->getLastSerial()); - prev = it; +FileChunk::TmpChunkMeta::fill(vespalib::nbostream & is) { + resize(getNumEntries()); + for (LidMeta & lm : *this) { + lm.deserialize(is); } } -} - void FileChunk::erase() { @@ -152,98 +131,92 @@ FileChunk::erase() std::filesystem::remove(std::filesystem::path(_dataFileName)); } -size_t +void FileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit) { - size_t sz(0); assert(_chunkInfo.empty()); FastOS_File idxFile(_idxFileName.c_str()); idxFile.enableMemoryMap(0); - if (idxFile.OpenReadOnly()) { - if (idxFile.IsMemoryMapped()) { - const int64_t fileSize = idxFile.getSize(); - if (_idxHeaderLen == 0) { - _idxHeaderLen = readIdxHeader(idxFile, _docIdLimit); + if ( ! idxFile.OpenReadOnly()) { + LOG_ABORT("should not reach here"); + } + if ( ! idxFile.IsMemoryMapped()) { + assert(idxFile.getSize() == 0); + return; + } + const int64_t fileSize = idxFile.getSize(); + if (_idxHeaderLen == 0) { + _idxHeaderLen = readIdxHeader(idxFile, _docIdLimit); + } + BucketDensityComputer globalBucketMap(_bucketizer); + // Guard comes from the same bucketizer so the same guard can be used + // for both local and global BucketDensityComputer + vespalib::GenerationHandler::Guard bucketizerGuard = globalBucketMap.getGuard(); + vespalib::nbostream is(static_cast<const char *>(idxFile.MemoryMapPtr(0)) + _idxHeaderLen, + fileSize - _idxHeaderLen); + for (size_t count=0; ! is.empty() && is.good(); count++) { + const int64_t lastKnownGoodPos = _idxHeaderLen + is.rp(); + TmpChunkMeta chunkMeta; + try { + chunkMeta.deserialize(is); + chunkMeta.fill(is); + if ((count == 0) && (chunkMeta.getLastSerial() < serialNum)) { + LOG(warning, "last serial num(%" PRIu64 ") from previous file is bigger than my first(%" PRIu64 + "). That is odd.Current filename is '%s'", + serialNum, chunkMeta.getLastSerial(), _idxFileName.c_str()); + serialNum = chunkMeta.getLastSerial(); } - vespalib::nbostream is(static_cast<const char *>(idxFile.MemoryMapPtr(0)) + _idxHeaderLen, - fileSize - _idxHeaderLen); - TmpChunkMetaV tempVector; - tempVector.reserve(fileSize/(sizeof(ChunkMeta)+sizeof(LidMeta))); - while ( ! is.empty() && is.good()) { - const int64_t lastKnownGoodPos = _idxHeaderLen + is.rp(); - tempVector.emplace_back(); - TmpChunkMeta & chunkMeta(tempVector.back()); - try { - chunkMeta.deserialize(is); - chunkMeta.fill(is); - } catch (const vespalib::IllegalStateException & e) { - LOG(warning, "Exception deserializing idx file : %s", e.what()); - LOG(warning, "File '%s' seems to be partially truncated. Will truncate from size=%" PRId64 " to %" PRId64, - _idxFileName.c_str(), fileSize, lastKnownGoodPos); - FastOS_File toTruncate(_idxFileName.c_str()); - if ( toTruncate.OpenReadWrite()) { - if (toTruncate.SetSize(lastKnownGoodPos)) { - tempVector.resize(tempVector.size() - 1); - } else { - throw SummaryException("SetSize() failed.", toTruncate, VESPA_STRLOC); - } - } else { - throw SummaryException("Open for truncation failed.", toTruncate, VESPA_STRLOC); - } - break; + assert(serialNum <= chunkMeta.getLastSerial()); + serialNum = handleChunk(guard, ds, docIdLimit, bucketizerGuard, globalBucketMap, chunkMeta); + assert(serialNum >= _lastPersistedSerialNum.load(std::memory_order_relaxed)); + _lastPersistedSerialNum.store(serialNum, std::memory_order_relaxed); + } catch (const vespalib::IllegalStateException & e) { + LOG(warning, "Exception deserializing idx file : %s", e.what()); + LOG(warning, "File '%s' seems to be partially truncated. Will truncate from size=%" PRId64 " to %" PRId64, + _idxFileName.c_str(), fileSize, lastKnownGoodPos); + FastOS_File toTruncate(_idxFileName.c_str()); + if ( toTruncate.OpenReadWrite()) { + if (toTruncate.SetSize(lastKnownGoodPos)) { + } else { + throw SummaryException("SetSize() failed.", toTruncate, VESPA_STRLOC); } + } else { + throw SummaryException("Open for truncation failed.", toTruncate, VESPA_STRLOC); } - if ( ! tempVector.empty()) { - verifyOrAssert(tempVector); - if (tempVector[0].getLastSerial() < serialNum) { - LOG(warning, - "last serial num(%" PRIu64 ") from previous file is " - "bigger than my first(%" PRIu64 "). That is odd." - "Current filename is '%s'", - serialNum, tempVector[0].getLastSerial(), - _idxFileName.c_str()); - serialNum = tempVector[0].getLastSerial(); - } - BucketDensityComputer globalBucketMap(_bucketizer); - // Guard comes from the same bucketizer so the same guard can be used - // for both local and global BucketDensityComputer - vespalib::GenerationHandler::Guard bucketizerGuard = globalBucketMap.getGuard(); - for (const TmpChunkMeta & chunkMeta : tempVector) { - assert(serialNum <= chunkMeta.getLastSerial()); - BucketDensityComputer bucketMap(_bucketizer); - for (size_t i(0), m(chunkMeta.getNumEntries()); i < m; i++) { - const LidMeta & lidMeta(chunkMeta[i]); - if (lidMeta.getLid() < docIdLimit) { - if (_bucketizer && (lidMeta.size() > 0)) { - document::BucketId bucketId = _bucketizer->getBucketOf(bucketizerGuard, lidMeta.getLid()); - bucketMap.recordLid(bucketId); - globalBucketMap.recordLid(bucketId); - } - ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size())); - _numLids++; - } else { - remove(lidMeta.getLid(), lidMeta.size()); - } - _addedBytes += adjustSize(lidMeta.size()); - } - serialNum = chunkMeta.getLastSerial(); - addNumBuckets(bucketMap.getNumBuckets()); - _chunkInfo.emplace_back(chunkMeta.getOffset(), chunkMeta.getSize(), chunkMeta.getLastSerial()); - assert(serialNum >= _lastPersistedSerialNum.load(std::memory_order_relaxed)); - _lastPersistedSerialNum.store(serialNum, std::memory_order_relaxed); - } - _numUniqueBuckets = globalBucketMap.getNumBuckets(); + break; + } + } + _numUniqueBuckets = globalBucketMap.getNumBuckets(); +} + +uint64_t +FileChunk::handleChunk(const unique_lock &guard, ISetLid &ds, uint32_t docIdLimit, + const vespalib::GenerationHandler::Guard & bucketizerGuard, BucketDensityComputer &globalBucketMap, + const TmpChunkMeta & chunkMeta) { + BucketDensityComputer bucketMap(_bucketizer); + for (size_t i(0), m(chunkMeta.getNumEntries()); i < m; i++) { + const LidMeta & lidMeta(chunkMeta[i]); + if (lidMeta.getLid() < docIdLimit) { + if (_bucketizer && (lidMeta.size() > 0)) { + document::BucketId bucketId = _bucketizer->getBucketOf(bucketizerGuard, lidMeta.getLid()); + bucketMap.recordLid(bucketId); + globalBucketMap.recordLid(bucketId); } + ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size())); + _numLids++; } else { - assert(idxFile.getSize() == 0); + remove(lidMeta.getLid(), lidMeta.size()); } - } else { - LOG_ABORT("should not reach here"); + _addedBytes += adjustSize(lidMeta.size()); } - return sz; + uint64_t serialNum = chunkMeta.getLastSerial(); + addNumBuckets(bucketMap.getNumBuckets()); + _chunkInfo.emplace_back(chunkMeta.getOffset(), chunkMeta.getSize(), chunkMeta.getLastSerial()); + return serialNum; } + void FileChunk::enableRead() { @@ -581,8 +554,7 @@ FileChunk::getStats() const uint64_t serialNum = getLastPersistedSerialNum(); uint32_t docIdLimit = getDocIdLimit(); uint64_t nameId = getNameId().getId(); - return DataStoreFileChunkStats(diskFootprint, diskBloat, bucketSpread, - serialNum, serialNum, docIdLimit, nameId); + return {diskFootprint, diskBloat, bucketSpread, serialNum, serialNum, docIdLimit, nameId}; } } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index 3664da3dfd9..446a53de446 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -47,7 +47,7 @@ public: class BucketDensityComputer { public: - BucketDensityComputer(const IBucketizer * bucketizer) : _bucketizer(bucketizer), _count(0) { } + explicit BucketDensityComputer(const IBucketizer * bucketizer) : _bucketizer(bucketizer), _count(0) { } void recordLid(const vespalib::GenerationHandler::Guard & guard, uint32_t lid, uint32_t dataSize) { if (_bucketizer && (dataSize > 0)) { recordLid(_bucketizer->getBucketOf(guard, lid)); @@ -109,7 +109,7 @@ public: const IBucketizer *bucketizer); virtual ~FileChunk(); - virtual size_t updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit); + virtual void updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit); virtual ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const; virtual void read(LidInfoWithLidV::const_iterator begin, size_t count, IBufferVisitor & visitor) const; void remove(uint32_t lid, uint32_t size); @@ -118,7 +118,7 @@ public: virtual size_t getMemoryMetaFootprint() const; virtual vespalib::MemoryUsage getMemoryUsage() const; - virtual size_t getDiskHeaderFootprint(void) const { return _dataHeaderLen + _idxHeaderLen; } + virtual size_t getDiskHeaderFootprint() const { return _dataHeaderLen + _idxHeaderLen; } size_t getDiskBloat() const { return (_addedBytes == 0) ? getDiskFootprint() @@ -199,6 +199,16 @@ public: static vespalib::string createIdxFileName(const vespalib::string & name); static vespalib::string createDatFileName(const vespalib::string & name); private: + class TmpChunkMeta : public ChunkMeta, + public std::vector<LidMeta> + { + public: + void fill(vespalib::nbostream & is); + }; + using BucketizerGuard = vespalib::GenerationHandler::Guard; + uint64_t handleChunk(const unique_lock &guard, ISetLid &lidMap, uint32_t docIdLimit, + const BucketizerGuard & bucketizerGuard, BucketDensityComputer & global, + const TmpChunkMeta & chunkMeta); using File = std::unique_ptr<FileRandRead>; const FileId _fileId; const NameId _nameId; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 7102b80d7d0..973287fc7bd 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -174,16 +174,15 @@ WriteableFileChunk::~WriteableFileChunk() } } -size_t +void WriteableFileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit) { - size_t sz = FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit); + FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit); _nextChunkId = _chunkInfo.size(); _active = std::make_unique<Chunk>(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes())); _serialNum = getLastPersistedSerialNum(); _firstChunkIdToBeWritten = _active->getId(); setDiskFootprint(0); - return sz; } void diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index b5a52dc83f7..028915d28e0 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -64,7 +64,7 @@ public: size_t getMemoryFootprint() const override; size_t getMemoryMetaFootprint() const override; vespalib::MemoryUsage getMemoryUsage() const override; - size_t updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit) override; + void updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit) override; void waitForDiskToCatchUpToNow() const; void flushPendingChunks(uint64_t serialNum); DataStoreFileChunkStats getStats() const override; |