diff options
43 files changed, 342 insertions, 122 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 94b456b3f5e..300a55e521a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -156,7 +156,7 @@ public class SummaryClass extends Derived { summaryField.getTransform() == SummaryTransform.POSITIONS || summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER || - summaryField.getTransform() == SummaryTransform.LINGUISTICS_TOKENS) + summaryField.getTransform() == SummaryTransform.TOKENS) { return summaryField.getSingleSource(); } else if (summaryField.getTransform().isDynamic()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index 54a4883fa00..2f60cd8eb06 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -92,7 +92,7 @@ public class SummaryClassField { return Type.FEATUREDATA; } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { return Type.FEATUREDATA; - } else if (transform != null && transform.equals(SummaryTransform.LINGUISTICS_TOKENS)) { + } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) { return Type.JSONSTRING; } else { return Type.LONGSTRING; diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 61f68defe40..ffb86f8ecf2 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -217,8 +217,8 @@ public class ConvertParsedFields { transform = SummaryTransform.MATCHED_ELEMENTS_FILTER; } else if (parsed.getDynamic()) { transform = SummaryTransform.DYNAMICTEASER; - } else if (parsed.getLinguisticsTokens()) { - transform = SummaryTransform.LINGUISTICS_TOKENS; + } else if (parsed.getTokens()) { + transform = SummaryTransform.TOKENS; } if (parsed.getBolded()) { transform = transform.bold(); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 9145934501c..6ca7537205c 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -150,6 +150,12 @@ public class ConvertParsedSchemas { var parsedType = parsedField.getType(); DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null; var existingField = schema.getField(parsedField.name()); + if (existingField == null && parsedField.getSources().size() == 1) { + var sourceName = parsedField.getSources().get(0); + if (!sourceName.equals(parsedField.name())) { + existingField = schema.getField(sourceName); + } + } if (existingField != null) { var existingType = existingField.getDataType(); if (dataType == null) { diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java index 446981f1ba4..8b732c358f5 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java @@ -18,7 +18,7 @@ class ParsedSummaryField extends ParsedBlock { private boolean isMEO = false; private boolean isFull = false; private boolean isBold = false; - private boolean isLinguisticsTokens = false; + private boolean isTokens = false; private final List<String> sources = new ArrayList<>(); private final List<String> destinations = new ArrayList<>(); @@ -38,7 +38,7 @@ class ParsedSummaryField extends ParsedBlock { boolean getDynamic() { return isDyn; } boolean getFull() { return isFull; } boolean getMatchedElementsOnly() { return isMEO; } - boolean getLinguisticsTokens() { return isLinguisticsTokens; } + boolean getTokens() { return isTokens; } void addDestination(String dst) { destinations.add(dst); } void addSource(String src) { sources.add(src); } @@ -46,7 +46,7 @@ class ParsedSummaryField extends ParsedBlock { void setDynamic() { this.isDyn = true; } void setFull() { this.isFull = true; } void setMatchedElementsOnly() { this.isMEO = true; } - void setLinguisticsTokens() { this.isLinguisticsTokens = true; } + void setTokens() { this.isTokens = true; } void setType(ParsedType value) { verifyThat(type == null, "Cannot change type from ", type, "to", value); this.type = value; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index e54f8d3e881..e4116c3f9d5 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -79,7 +79,7 @@ public class IndexingOutputs extends Processor { } dynamicSummary.add(summaryName); } else if (summaryTransform != SummaryTransform.ATTRIBUTE && - summaryTransform != SummaryTransform.LINGUISTICS_TOKENS) { + summaryTransform != SummaryTransform.TOKENS) { staticSummary.add(summaryName); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index c7c1606951e..50be01db04b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -24,7 +24,7 @@ public enum SummaryTransform { MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), COPY("copy"), DOCUMENT_ID("documentid"), - LINGUISTICS_TOKENS("linguistics-tokens"); + TOKENS("tokens"); private final String name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index f434d056bfc..07952c09b6d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -49,10 +49,10 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; +import static java.util.logging.Level.FINE; /** * A container cluster that is typically set up from the user application. @@ -218,8 +218,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCost.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(Level.FINE, () -> "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); + logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" + .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); } return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index aa4aa6af32c..4e3b3d1d8cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -109,7 +109,7 @@ public abstract class Container extends AbstractService implements addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags())); - addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x200000"); + addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x800000"); } protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index a454c1141ca..9729d7d806b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -14,18 +14,16 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; - import com.yahoo.yolean.Exceptions; -import java.io.File; import java.io.Serializable; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; +import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; /** @@ -163,7 +161,7 @@ public class UserConfiguredFiles implements Serializable { ApplicationFile file = applicationPackage.getFile(path); if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty())) - logger.logApplicationPackage(WARNING, "Directory '" + path.getRelative() + "' is empty"); + logger.logApplicationPackage(INFO, "Directory '" + path.getRelative() + "' is empty"); FileReference reference = registeredFiles.get(path); if (reference == null) { diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index a5238afc86a..f186caacb5f 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -201,7 +201,7 @@ TOKEN : | < FULL: "full" > | < STATIC: "static" > | < DYNAMIC: "dynamic" > -| < LINGUISTICS_TOKENS: "linguistics-tokens" > +| < TOKENS: "tokens" > | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" > | < SSCONTEXTUAL: "contextual" > | < SSOVERRIDE: "override" > @@ -1129,7 +1129,7 @@ void summaryInFieldShort(ParsedField field) : <COLON> ( <DYNAMIC> { psf.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { psf.setFull(); } - | <LINGUISTICS_TOKENS> { psf.setLinguisticsTokens(); } + | <TOKENS> { psf.setTokens(); } ) } @@ -1175,7 +1175,7 @@ void summaryTransform(ParsedSummaryField field) : { } ( <DYNAMIC> { field.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { field.setFull(); } - | <LINGUISTICS_TOKENS> { field.setLinguisticsTokens(); } + | <TOKENS> { field.setTokens(); } ) } @@ -2715,7 +2715,6 @@ String identifier() : { } | <INLINE> | <INPUTS> | <INTEGER> - | <LINGUISTICS_TOKENS> | <LITERAL> | <LOCALE> | <LONG> @@ -2769,6 +2768,7 @@ String identifier() : { } | <TERTIARY> | <TEXT> | <TO> + | <TOKENS> | <TRUE> | <TYPE> | <UCA> diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index 4128baddcb7..a1d726473be 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -227,16 +227,16 @@ public class SummaryTestCase extends AbstractSchemaTestCase { } @Test - void linguistics_tokenizer_override() throws ParseException { + void tokens_override() throws ParseException { var schema = buildSchema("field foo type string { indexing: summary }", joinLines("document-summary bar {", - " summary baz type string {", + " summary baz {", " source: foo ", - " linguistics-tokens", + " tokens", " }", " from-disk", "}")); - assertOverride(schema, "baz", SummaryTransform.LINGUISTICS_TOKENS.getName(), "foo", "bar"); + assertOverride(schema, "baz", SummaryTransform.TOKENS.getName(), "foo", "bar"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index b4a54548062..3dd845ec56f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.schema.processing.ReservedRankingExpressionFunctionNamesTestCase; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -23,7 +22,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.HashMap; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index f406095d579..fd4a34118c5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -14,9 +14,12 @@ import com.yahoo.yolean.concurrent.Memoized; import java.io.InputStream; import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; import static java.util.Objects.requireNonNull; @@ -28,6 +31,8 @@ import static java.util.Objects.requireNonNull; */ public class DeploymentData { + private static final Logger log = Logger.getLogger(DeploymentData.class.getName()); + private final ApplicationId instance; private final ZoneId zone; private final Supplier<InputStream> applicationPackage; @@ -56,14 +61,14 @@ public class DeploymentData { this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); this.platform = requireNonNull(platform); - this.endpoints = new Memoized<>(requireNonNull(endpoints)); + this.endpoints = wrap(requireNonNull(endpoints), Duration.ofSeconds(30), "deployment endpoints for " + instance + " in " + zone); this.dockerImageRepo = requireNonNull(dockerImageRepo); this.athenzDomain = athenzDomain; - this.quota = new Memoized<>(requireNonNull(quota)); + this.quota = wrap(requireNonNull(quota), Duration.ofSeconds(10), "quota for " + instance); this.tenantSecretStores = List.copyOf(requireNonNull(tenantSecretStores)); this.operatorCertificates = List.copyOf(requireNonNull(operatorCertificates)); - this.cloudAccount = new Memoized<>(requireNonNull(cloudAccount)); - this.dataPlaneTokens = new Memoized<>(dataPlaneTokens); + this.cloudAccount = wrap(requireNonNull(cloudAccount), Duration.ofSeconds(5), "cloud account for " + instance + " in " + zone); + this.dataPlaneTokens = wrap(dataPlaneTokens, Duration.ofSeconds(5), "data plane tokens for " + instance + " in " + zone); this.dryRun = dryRun; } @@ -83,8 +88,8 @@ public class DeploymentData { return platform; } - public Supplier<DeploymentEndpoints> endpoints() { - return endpoints; + public DeploymentEndpoints endpoints() { + return endpoints.get(); } public Optional<DockerImage> dockerImageRepo() { @@ -119,4 +124,41 @@ public class DeploymentData { return dryRun; } + private static <T> Supplier<T> wrap(Supplier<T> delegate, Duration timeout, String description) { + return new TimingSupplier<>(new Memoized<>(delegate), timeout, description); + } + + public static class TimingSupplier<T> implements Supplier<T> { + + private final Supplier<T> delegate; + private final Duration timeout; + private final String description; + + public TimingSupplier(Supplier<T> delegate, Duration timeout, String description) { + this.delegate = delegate; + this.timeout = timeout; + this.description = description; + } + + @Override + public T get() { + long startNanos = System.nanoTime(); + Throwable thrown = null; + try { + return delegate.get(); + } + catch (Throwable t) { + thrown = t; + throw t; + } + finally { + long durationNanos = System.nanoTime() - startNanos; + Level level = durationNanos > timeout.toNanos() ? Level.WARNING : Level.FINE; + String thrownMessage = thrown == null ? "" : " with exception " + thrown; + log.log(level, () -> String.format("Getting %s took %.6f seconds%s", description, durationNanos / 1e9, thrownMessage)); + } + } + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java index 421ec99d6f2..13fa6c862a7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java @@ -67,7 +67,7 @@ public class EndpointCertificateValidatorImpl implements EndpointCertificateVali } catch (SecretNotFoundException s) { // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner - throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); + throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store", s); } catch (EndpointCertificateException e) { if (!e.type().equals(EndpointCertificateException.Type.CERT_NOT_AVAILABLE)) { // such failures are normal and will be retried, it takes some time to show up in the secret store log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 96fa6527cf4..35d09fd541b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -93,6 +94,8 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}") + .get(self::accountantTenant)) .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/preview") .get(self::previewBill) .post(Slime.class, self::createBill)) @@ -430,6 +433,23 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return slime; } + private Slime accountantTenant(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var slime = new Slime(); + var root = slime.setObject(); + + var planId = billing.getPlan(tenant.name()); + var plan = planRegistry.plan(planId); + + var collection = billing.getCollectionMethod(tenant.name()); + + toSlime(root, tenant, planId, plan, collection); + + return slime; + } + // --------- INVOICE RENDERING ---------- private void invoicesSummaryToSlime(Cursor slime, List<Bill> bills) { @@ -501,6 +521,33 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler cost.ifPresent(c -> slime.setString("cost", c.toString())); } + private void toSlime(Cursor slime, CloudTenant tenant, PlanId planId, Optional<Plan> plan, CollectionMethod method) { + slime.setString("tenant", tenant.name().value()); + toSlime(slime.setObject("plan"), planId, plan); + toSlime(slime.setObject("billing"), tenant.billingReference()); + slime.setString("collection", method.name()); + } + + private void toSlime(Cursor slime, PlanId planId, Optional<Plan> plan) { + slime.setString("id", planId.value()); + if (plan.isPresent()) { + slime.setString("name", plan.get().displayName()); + slime.setBool("billed", plan.get().isBilled()); + slime.setBool("supported", plan.get().isSupported()); + } else { + slime.setString("name", "UNKNOWN"); + slime.setBool("billed", false); + slime.setBool("supported", false); + } + } + + private void toSlime(Cursor slime, Optional<BillingReference> billingReference) { + if (billingReference.isPresent()) { + slime.setString("id", billingReference.get().reference()); + slime.setLong("lastUpdated", billingReference.get().updated().toEpochMilli()); + } + } + private List<Object[]> toCsv(Bill bill) { return List.<Object[]>of(new Object[]{ bill.id().value(), bill.tenant().value(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 259e877afd9..5995b3eaac6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -417,12 +417,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer applications.put(id, new Application(id.applicationId(), lastPrepareVersion, appPackage)); ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); - deployment.endpoints(); // Supplier with side effects >_< if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); - this.containerEndpoints.put(id, deployment.endpoints().get().endpoints()); + this.containerEndpoints.put(id, deployment.endpoints().endpoints()); deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account)); if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index b7c86246158..4d3297ddb0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -237,4 +237,12 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { {"collection":"INVOICE"}"""); } } + + @Test + void require_accountant_tenant() { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes","billed":false,"supported":false},"billing":{},"collection":"AUTO"}"""); + } } diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 78c85607ea3..04512cd64d8 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -65,7 +65,7 @@ <assertj.vespa.version>3.24.2</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.vespa.version>1.11.43</athenz.vespa.version> + <athenz.vespa.version>1.11.44</athenz.vespa.version> <aws-sdk.vespa.version>1.12.565</aws-sdk.vespa.version> <!-- Athenz END --> diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index 6e09fa2de0f..051c5027999 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -10,8 +10,10 @@ #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/util/stringfmt.h> #include <charconv> +#ifdef __linux__ #include <malloc.h> #include <dlfcn.h> +#endif using vespalib::make_string_short::fmt; @@ -62,6 +64,7 @@ struct MemoryUsage { size_t malloc_current; }; +#ifdef __linux__ static const vespalib::string UNKNOWN = "unknown"; size_t convert(const vespalib::string & s) { @@ -94,7 +97,7 @@ MemoryUsage extract_memory_usage() { usage.rss_size = convert(vm_rss); #if __GLIBC_PREREQ(2, 33) - struct mallinfo2 mallocInfo = mallinfo2(); + struct mallinfo2 info = mallinfo2(); usage.malloc_peak = size_t(info.usmblks); usage.malloc_current = size_t(info.arena + info.hblkhd); #else @@ -111,6 +114,11 @@ MemoryUsage extract_memory_usage() { #endif return usage; } +#else +MemoryUsage extract_memory_usage() { + return { 0, 0, 0, 0 }; +} +#endif void report_memory_usage(const vespalib::string &desc) { MemoryUsage m = extract_memory_usage(); diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java index c56dd219879..21c189e2549 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java @@ -23,7 +23,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * Used to create builders for multi part http body entities, which stream their data. + * Used to create builders for multipart HTTP body entities, which stream their data. * * @author jonmv */ diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 232ecd0e3e5..729150bcef1 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -284,9 +284,6 @@ <exclude>org.junit.platform:junit-platform-engine:(${junit.platform.vespa.tenant.version},):jar:*</exclude> <exclude>org.junit.platform:junit-platform-commons:(,${junit.platform.vespa.tenant.version}):jar:*</exclude> <exclude>org.junit.platform:junit-platform-commons:(${junit.platform.vespa.tenant.version},):jar:*</exclude> - - <!-- Ban testng as its presence breaks the Java based test framework for Cloud --> - <exclude>org.testng:testng:*:jar:test</exclude> </excludes> </bannedDependencies> </rules> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index c722779c3c6..c65f2abb6fd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -84,6 +84,13 @@ public class SyncFileInfo { } else if (filename.startsWith("start-services.out-")) { compression = Compression.ZSTD; dir = "logs/start-services/"; + } else if (filename.startsWith("nginx-error")) { + compression = Compression.ZSTD; + if ("nginx-error.log".equals(filename)) { + if (!rotatedOnly) remoteFilename = "nginx-error.log"; + minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO; + } + dir = "logs/nginx/"; } else { compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; if (rotatedOnly && compression != Compression.NONE) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index f3a25778459..8e56741274e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -40,6 +40,9 @@ public class SyncFileInfoTest { private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out"); private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143"); + private static final Path rotatedNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log.20231019-1234555"); + private static final Path currentNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log"); + private static final Path nginxAccessLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-access.log.20231019-1234"); @Test void access_logs() { @@ -96,6 +99,22 @@ public class SyncFileInfoTest { } @Test + void nginx_error_logs() { + new UnixPath(currentNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ofHours(1),true); + assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ZERO,false); + + new UnixPath(rotatedNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, true); + assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, false); + + // Does not sync access logs + new UnixPath(nginxAccessLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, nginxAccessLog, false, ApplicationId.defaultId()); + assertEquals(Optional.empty(), sfi); + } + + @Test void start_services() { assertForLogFile(startServicesPath1, null, null, true); assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index b4968c66a67..264e981558a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -222,6 +222,14 @@ public record IP() { this.hostnames = List.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); } + /** The number of hosts in this pool: each host has a name and/or one or two IP addresses. */ + public long size() { + return hostnames().isEmpty() ? + Math.max(ipAddresses.addresses.stream().filter(IP::isV4).count(), + ipAddresses.addresses.stream().filter(IP::isV6).count()) : + hostnames().size(); + } + public List<String> ips() { return ipAddresses.addresses; } /** diff --git a/parent/pom.xml b/parent/pom.xml index aed5fe071f3..52062f5897d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1121,7 +1121,7 @@ See pluginManagement of rewrite-maven-plugin for more details --> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-recipe-bom</artifactId> - <version>2.3.1</version> + <version>2.4.0</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index 64cbde68416..bcf9848ff66 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchcore_flushengine STATIC flushtargetproxy.cpp flushtask.cpp prepare_restart_flush_strategy.cpp + priority_flush_token.cpp threadedflushtarget.cpp tls_stats_factory.cpp tls_stats_map.cpp diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index fc08d4d8a17..768800ee781 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -66,16 +66,18 @@ FlushEngine::FlushMeta::~FlushMeta() = default; FlushEngine::FlushInfo::FlushInfo() : FlushMeta("", "", 0), - _target() + _target(), + _priority_flush_token() { } FlushEngine::FlushInfo::~FlushInfo() = default; -FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP& target) +FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP& target, std::shared_ptr<PriorityFlushToken> priority_flush_token) : FlushMeta(handler_name, target->getName(), taskId), - _target(target) + _target(target), + _priority_flush_token(std::move(priority_flush_token)) { } @@ -89,6 +91,7 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStats _has_thread(false), _strategy(std::move(strategy)), _priorityStrategy(), + _priority_flush_token(), _executor(maxConcurrentTotal(), CpuUsage::wrap(flush_engine_executor, CpuUsage::Category::COMPACT)), _lock(), _cond(), @@ -249,7 +252,7 @@ createName(const IFlushHandler &handler, const vespalib::string &targetName) bool FlushEngine::prune() { - std::set<IFlushHandler::SP> toPrune; + PendingPrunes toPrune; { std::lock_guard<std::mutex> guard(_lock); if (_pendingPrune.empty()) { @@ -257,7 +260,8 @@ FlushEngine::prune() } _pendingPrune.swap(toPrune); } - for (const auto &handler : toPrune) { + for (const auto& kv : toPrune) { + const auto& handler = kv.first; IFlushTarget::List lst = handler->getFlushTargets(); auto oldestFlushed = findOldestFlushedTarget(lst, *handler); if (LOG_WOULD_LOG(event)) { @@ -368,21 +372,21 @@ FlushEngine::initNextFlush(const FlushContext::List &lst) void FlushEngine::flushAll(const FlushContext::List &lst) { + mark_currently_flushing_tasks(_priority_flush_token); LOG(debug, "%ld targets to flush.", lst.size()); for (const FlushContext::SP & ctx : lst) { if (wait_for_slot(IFlushTarget::Priority::NORMAL)) { if (ctx->initFlush(get_flush_token(*ctx))) { logTarget("initiated", *ctx); - _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); + _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx, _priority_flush_token), *this, ctx)); } else { logTarget("failed to initiate", *ctx); } } } - _executor.sync(); - prune(); std::lock_guard<std::mutex> strategyGuard(_strategyLock); _priorityStrategy.reset(); + _priority_flush_token.reset(); _strategyCond.notify_all(); } @@ -404,19 +408,19 @@ FlushEngine::flushNextTarget(const vespalib::string & name, const FlushContext:: name.c_str(), contexts.size()); std::this_thread::sleep_for(100ms); } - _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); + _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx, {}), *this, ctx)); return ctx->getName(); } uint32_t -FlushEngine::initFlush(const FlushContext &ctx) +FlushEngine::initFlush(const FlushContext &ctx, std::shared_ptr<PriorityFlushToken> priority_flush_token) { if (LOG_WOULD_LOG(event)) { IFlushTarget::MemoryGain mgain(ctx.getTarget()->getApproxMemoryGain()); EventLogger::flushStart(ctx.getName(), mgain.getBefore(), mgain.getAfter(), mgain.gain(), ctx.getTarget()->getFlushedSerialNum() + 1, ctx.getHandler()->getCurrentSerialNumber()); } - return initFlush(ctx.getHandler(), ctx.getTarget()); + return initFlush(ctx.getHandler(), ctx.getTarget(), std::move(priority_flush_token)); } void @@ -434,10 +438,25 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) } LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, vespalib::to_s(duration)); std::lock_guard<std::mutex> guard(_lock); - _flushing.erase(taskId); + /* + * Hand over any priority flush token for completed flush to + * _pendingPrune, to ensure that setStrategy will wait until + * flush engine has called prune(). + */ + std::shared_ptr<PriorityFlushToken> priority_flush_token; + { + auto itr = _flushing.find(taskId); + if (itr != _flushing.end()) { + priority_flush_token = std::move(itr->second._priority_flush_token); + _flushing.erase(itr); + } + } assert(ctx.getHandler()); if (_handlers.hasHandler(ctx.getHandler())) { - _pendingPrune.insert(ctx.getHandler()); + auto ins_res = _pendingPrune.emplace(ctx.getHandler(), PendingPrunes::mapped_type()); + if (priority_flush_token) { + ins_res.first->second = std::move(priority_flush_token); + } } _cond.notify_all(); } @@ -450,7 +469,7 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler if (result) { _pendingPrune.erase(result); } - _pendingPrune.insert(flushHandler); + _pendingPrune.emplace(flushHandler, PendingPrunes::mapped_type()); return result; } @@ -475,13 +494,13 @@ FlushEngine::getCurrentlyFlushingSet() const } uint32_t -FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target) +FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target, std::shared_ptr<PriorityFlushToken> priority_flush_token) { uint32_t taskId; { std::lock_guard<std::mutex> guard(_lock); taskId = _taskId++; - FlushInfo flush(taskId, handler->getName(), target); + FlushInfo flush(taskId, handler->getName(), target, std::move(priority_flush_token)); _flushing[taskId] = flush; } LOG(debug, "FlushEngine::initFlush(handler='%s', target='%s') => taskId='%d'", @@ -492,6 +511,9 @@ FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP void FlushEngine::setStrategy(IFlushStrategy::SP strategy) { + std::promise<void> promise; + auto future = promise.get_future(); + auto priority_flush_token = std::make_shared<PriorityFlushToken>(std::move(promise)); std::lock_guard<std::mutex> setStrategyGuard(_setStrategyLock); std::unique_lock<std::mutex> strategyGuard(_strategyLock); if (_closed.load(std::memory_order_relaxed)) { @@ -499,6 +521,7 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy) } assert(!_priorityStrategy); _priorityStrategy = std::move(strategy); + _priority_flush_token = std::move(priority_flush_token); { std::lock_guard<std::mutex> guard(_lock); _cond.notify_all(); @@ -506,6 +529,22 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy) while (_priorityStrategy) { _strategyCond.wait(strategyGuard); } + strategyGuard.unlock(); + /* + * Wait for flushes started before the strategy change, for + * flushes initiated by the strategy, and for flush engine to call + * prune() afterwards. + */ + future.wait(); +} + +void +FlushEngine::mark_currently_flushing_tasks(std::shared_ptr<PriorityFlushToken> priority_flush_token) +{ + std::lock_guard<std::mutex> guard(_lock); + for (auto& kv : _flushing) { + kv.second._priority_flush_token = priority_flush_token; + } } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index ec0f019f6e4..302c4a2499e 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -3,6 +3,7 @@ #include "flushcontext.h" #include "iflushstrategy.h" +#include "priority_flush_token.h" #include <vespa/searchcore/proton/common/handlermap.hpp> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/vespalib/util/threadstackexecutor.h> @@ -43,13 +44,15 @@ private: struct FlushInfo : public FlushMeta { FlushInfo(); - FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP &target); + FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP &target, std::shared_ptr<PriorityFlushToken> priority_flush_token); ~FlushInfo(); IFlushTarget::SP _target; + std::shared_ptr<PriorityFlushToken> _priority_flush_token; }; using FlushMap = std::map<uint32_t, FlushInfo>; using FlushHandlerMap = HandlerMap<IFlushHandler>; + using PendingPrunes = std::map<std::shared_ptr<IFlushHandler>, std::shared_ptr<PriorityFlushToken>>; std::atomic<bool> _closed; const uint32_t _maxConcurrentNormal; const vespalib::duration _idleInterval; @@ -58,6 +61,7 @@ private: std::atomic<bool> _has_thread; IFlushStrategy::SP _strategy; mutable IFlushStrategy::SP _priorityStrategy; + mutable std::shared_ptr<PriorityFlushToken> _priority_flush_token; vespalib::ThreadStackExecutor _executor; mutable std::mutex _lock; std::condition_variable _cond; @@ -67,7 +71,7 @@ private: std::mutex _strategyLock; std::condition_variable _strategyCond; std::shared_ptr<flushengine::ITlsStatsFactory> _tlsStatsFactory; - std::set<IFlushHandler::SP> _pendingPrune; + PendingPrunes _pendingPrune; std::shared_ptr<search::FlushToken> _normal_flush_token; std::shared_ptr<search::FlushToken> _gc_flush_token; @@ -78,8 +82,8 @@ private: vespalib::string flushNextTarget(const vespalib::string & name, const FlushContext::List & contexts); void flushAll(const FlushContext::List &lst); bool prune(); - uint32_t initFlush(const FlushContext &ctx); - uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target); + uint32_t initFlush(const FlushContext &ctx, std::shared_ptr<PriorityFlushToken> priority_flush_token); + uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target, std::shared_ptr<PriorityFlushToken> priority_flush_token); void flushDone(const FlushContext &ctx, uint32_t taskId); bool canFlushMore(const std::unique_lock<std::mutex> &guard, IFlushTarget::Priority priority) const; void wait_for_slot_or_pending_prune(IFlushTarget::Priority priority); @@ -179,6 +183,7 @@ public: FlushMetaSet getCurrentlyFlushingSet() const; void setStrategy(IFlushStrategy::SP strategy); + void mark_currently_flushing_tasks(std::shared_ptr<PriorityFlushToken> priority_flush_token); uint32_t maxConcurrentTotal() const { return _maxConcurrentNormal + 1; } uint32_t maxConcurrentNormal() const { return _maxConcurrentNormal; } }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.cpp new file mode 100644 index 00000000000..f031c19d1d8 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.cpp @@ -0,0 +1,17 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "priority_flush_token.h" + +namespace proton { + +PriorityFlushToken::PriorityFlushToken(std::promise<void> promise) + : _promise(std::move(promise)) +{ +} + +PriorityFlushToken::~PriorityFlushToken() +{ + _promise.set_value(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.h b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.h new file mode 100644 index 00000000000..82048b3eb3f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.h @@ -0,0 +1,21 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/idestructorcallback.h> +#include <future> + +namespace proton { + +/* + * This token is shared between flushes initiated from a priority flush + * strategy (cf. Proton::triggerFLush and Proton::prepareRestart). + */ +class PriorityFlushToken : public vespalib::IDestructorCallback { + std::promise<void> _promise; +public: + PriorityFlushToken(std::promise<void> promise); + ~PriorityFlushToken() override; +}; + +} diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index a091f8b5358..f771a8e4494 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -20,7 +20,7 @@ vespa_define_module( src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw src/tests/docsummary/document_id_dfw - src/tests/docsummary/linguistics_tokens_converter + src/tests/docsummary/tokens_converter src/tests/docsummary/matched_elements_filter src/tests/docsummary/query_term_filter_factory src/tests/docsummary/result_class diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt deleted file mode 100644 index d9510c3a2b3..00000000000 --- a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchsummary_linguistics_tokens_converter_test_app TEST - SOURCES - linguistics_tokens_converter_test.cpp - DEPENDS - searchsummary - GTest::gtest -) - -vespa_add_test(NAME searchsummary_linguistics_tokens_converter_test_app COMMAND searchsummary_linguistics_tokens_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt new file mode 100644 index 00000000000..68885a74b1b --- /dev/null +++ b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_tokens_converter_test_app TEST + SOURCES + tokens_converter_test.cpp + DEPENDS + searchsummary + GTest::gtest +) + +vespa_add_test(NAME searchsummary_tokens_converter_test_app COMMAND searchsummary_tokens_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp index beaa43c7af8..493cbe0ecba 100644 --- a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp +++ b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp @@ -10,7 +10,7 @@ #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/util/linguisticsannotation.h> #include <vespa/searchlib/util/token_extractor.h> -#include <vespa/searchsummary/docsummary/linguistics_tokens_converter.h> +#include <vespa/searchsummary/docsummary/tokens_converter.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/data/slime/slime.h> @@ -24,7 +24,7 @@ using document::Span; using document::SpanList; using document::SpanTree; using document::StringFieldValue; -using search::docsummary::LinguisticsTokensConverter; +using search::docsummary::TokensConverter; using search::linguistics::SPANTREE_NAME; using search::linguistics::TokenExtractor; using vespalib::SimpleBuffer; @@ -55,7 +55,7 @@ get_document_types_config() } -class LinguisticsTokensConverterTest : public testing::Test +class TokensConverterTest : public testing::Test { protected: std::shared_ptr<const DocumentTypeRepo> _repo; @@ -64,8 +64,8 @@ protected: vespalib::string _dummy_field_name; TokenExtractor _token_extractor; - LinguisticsTokensConverterTest(); - ~LinguisticsTokensConverterTest() override; + TokensConverterTest(); + ~TokensConverterTest() override; void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree); StringFieldValue make_annotated_string(bool alt_tokens); StringFieldValue make_annotated_chinese_string(); @@ -73,7 +73,7 @@ protected: vespalib::string convert(const StringFieldValue& fv); }; -LinguisticsTokensConverterTest::LinguisticsTokensConverterTest() +TokensConverterTest::TokensConverterTest() : testing::Test(), _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), _document_type(_repo->getDocumentType("indexingdocument")), @@ -83,10 +83,10 @@ LinguisticsTokensConverterTest::LinguisticsTokensConverterTest() { } -LinguisticsTokensConverterTest::~LinguisticsTokensConverterTest() = default; +TokensConverterTest::~TokensConverterTest() = default; void -LinguisticsTokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) +TokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) { StringFieldValue::SpanTrees trees; trees.push_back(std::move(tree)); @@ -94,7 +94,7 @@ LinguisticsTokensConverterTest::set_span_tree(StringFieldValue & value, std::uni } StringFieldValue -LinguisticsTokensConverterTest::make_annotated_string(bool alt_tokens) +TokensConverterTest::make_annotated_string(bool alt_tokens) { auto span_list_up = std::make_unique<SpanList>(); auto span_list = span_list_up.get(); @@ -111,7 +111,7 @@ LinguisticsTokensConverterTest::make_annotated_string(bool alt_tokens) } StringFieldValue -LinguisticsTokensConverterTest::make_annotated_chinese_string() +TokensConverterTest::make_annotated_chinese_string() { auto span_list_up = std::make_unique<SpanList>(); auto span_list = span_list_up.get(); @@ -125,50 +125,50 @@ LinguisticsTokensConverterTest::make_annotated_chinese_string() } vespalib::string -LinguisticsTokensConverterTest::make_exp_annotated_chinese_string_tokens() +TokensConverterTest::make_exp_annotated_chinese_string_tokens() { return R"(["我就是那个","大灰狼"])"; } vespalib::string -LinguisticsTokensConverterTest::convert(const StringFieldValue& fv) +TokensConverterTest::convert(const StringFieldValue& fv) { - LinguisticsTokensConverter converter(_token_extractor); + TokensConverter converter(_token_extractor); Slime slime; SlimeInserter inserter(slime); converter.convert(fv, inserter); return slime_to_string(slime); } -TEST_F(LinguisticsTokensConverterTest, convert_empty_string) +TEST_F(TokensConverterTest, convert_empty_string) { vespalib::string exp(R"([])"); StringFieldValue plain_string(""); EXPECT_EQ(exp, convert(plain_string)); } -TEST_F(LinguisticsTokensConverterTest, convert_plain_string) +TEST_F(TokensConverterTest, convert_plain_string) { vespalib::string exp(R"(["Foo Bar Baz"])"); StringFieldValue plain_string("Foo Bar Baz"); EXPECT_EQ(exp, convert(plain_string)); } -TEST_F(LinguisticsTokensConverterTest, convert_annotated_string) +TEST_F(TokensConverterTest, convert_annotated_string) { vespalib::string exp(R"(["foo","baz"])"); auto annotated_string = make_annotated_string(false); EXPECT_EQ(exp, convert(annotated_string)); } -TEST_F(LinguisticsTokensConverterTest, convert_annotated_string_with_alternatives) +TEST_F(TokensConverterTest, convert_annotated_string_with_alternatives) { vespalib::string exp(R"(["foo",["bar","baz"]])"); auto annotated_string = make_annotated_string(true); EXPECT_EQ(exp, convert(annotated_string)); } -TEST_F(LinguisticsTokensConverterTest, convert_annotated_chinese_string) +TEST_F(TokensConverterTest, convert_annotated_chinese_string) { auto exp = make_exp_annotated_chinese_string_tokens(); auto annotated_chinese_string = make_annotated_chinese_string(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 57b6004fb61..0287517f830 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -23,8 +23,6 @@ vespa_add_library(searchsummary_docsummary OBJECT juniper_dfw_term_visitor.cpp juniper_query_adapter.cpp juniperproperties.cpp - linguistics_tokens_converter.cpp - linguistics_tokens_dfw.cpp matched_elements_filter_dfw.cpp positionsdfw.cpp query_term_filter.cpp @@ -39,4 +37,6 @@ vespa_add_library(searchsummary_docsummary OBJECT struct_fields_resolver.cpp struct_map_attribute_combiner_dfw.cpp summaryfeaturesdfw.cpp + tokens_converter.cpp + tokens_dfw.cpp ) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp index c4823f6beeb..2ac5d1babbf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp @@ -12,12 +12,12 @@ const vespalib::string documentid("documentid"); const vespalib::string dynamic_teaser("dynamicteaser"); const vespalib::string empty("empty"); const vespalib::string geo_position("geopos"); -const vespalib::string linguistics_tokens("linguistics-tokens"); const vespalib::string matched_attribute_elements_filter("matchedattributeelementsfilter"); const vespalib::string matched_elements_filter("matchedelementsfilter"); const vespalib::string positions("positions"); const vespalib::string rank_features("rankfeatures"); const vespalib::string summary_features("summaryfeatures"); +const vespalib::string tokens("tokens"); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h index 2d0b8c23855..d53351d8b04 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h @@ -18,11 +18,11 @@ extern const vespalib::string documentid; extern const vespalib::string dynamic_teaser; extern const vespalib::string empty; extern const vespalib::string geo_position; -extern const vespalib::string linguistics_tokens; extern const vespalib::string matched_attribute_elements_filter; extern const vespalib::string matched_elements_filter; extern const vespalib::string positions; extern const vespalib::string rank_features; extern const vespalib::string summary_features; +extern const vespalib::string tokens; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp index d19d2994104..2f7d9acdb65 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -9,11 +9,11 @@ #include "geoposdfw.h" #include "idocsumenvironment.h" #include "juniperdfw.h" -#include "linguistics_tokens_dfw.h" #include "matched_elements_filter_dfw.h" #include "positionsdfw.h" #include "rankfeaturesdfw.h" #include "summaryfeaturesdfw.h" +#include "tokens_dfw.h" #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/util/exceptions.h> @@ -85,9 +85,9 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie } else { throw_missing_source(command); } - } else if (command == command::linguistics_tokens) { + } else if (command == command::tokens) { if (!source.empty()) { - fieldWriter = std::make_unique<LinguisticsTokensDFW>(source); + fieldWriter = std::make_unique<TokensDFW>(source); } else { throw_missing_source(command); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp index b9b9d7c4c97..e2849fe793e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp @@ -1,6 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "linguistics_tokens_converter.h" +#include "tokens_converter.h" #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/searchlib/util/token_extractor.h> #include <vespa/vespalib/data/slime/slime.h> @@ -14,18 +14,18 @@ using vespalib::slime::Inserter; namespace search::docsummary { -LinguisticsTokensConverter::LinguisticsTokensConverter(const TokenExtractor& token_extractor) +TokensConverter::TokensConverter(const TokenExtractor& token_extractor) : IStringFieldConverter(), _token_extractor(token_extractor), _text() { } -LinguisticsTokensConverter::~LinguisticsTokensConverter() = default; +TokensConverter::~TokensConverter() = default; template <typename ForwardIt> void -LinguisticsTokensConverter::handle_alternative_index_terms(ForwardIt it, ForwardIt last, Inserter& inserter) +TokensConverter::handle_alternative_index_terms(ForwardIt it, ForwardIt last, Inserter& inserter) { Cursor& a = inserter.insertArray(); ArrayInserter ai(a); @@ -35,13 +35,13 @@ LinguisticsTokensConverter::handle_alternative_index_terms(ForwardIt it, Forward } void -LinguisticsTokensConverter::handle_index_term(vespalib::stringref word, Inserter& inserter) +TokensConverter::handle_index_term(vespalib::stringref word, Inserter& inserter) { inserter.insertString(Memory(word)); } void -LinguisticsTokensConverter::handle_indexing_terms(const StringFieldValue& value, vespalib::slime::Inserter& inserter) +TokensConverter::handle_indexing_terms(const StringFieldValue& value, vespalib::slime::Inserter& inserter) { Cursor& a = inserter.insertArray(); ArrayInserter ai(a); @@ -63,14 +63,14 @@ LinguisticsTokensConverter::handle_indexing_terms(const StringFieldValue& value, } void -LinguisticsTokensConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter) +TokensConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter) { _text = input.getValueRef(); handle_indexing_terms(input, inserter); } bool -LinguisticsTokensConverter::render_weighted_set_as_array() const +TokensConverter::render_weighted_set_as_array() const { return true; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h index d752fe89ed9..1798abac203 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h @@ -10,10 +10,10 @@ namespace search::docsummary { /* * Class converting a string field value with annotations into an array - * containing the index terms. Multiple index terms at same position are + * containing the tokens. Multiple tokens at same position are * placed in a nested array. */ -class LinguisticsTokensConverter : public IStringFieldConverter +class TokensConverter : public IStringFieldConverter { const linguistics::TokenExtractor& _token_extractor; vespalib::stringref _text; @@ -23,8 +23,8 @@ class LinguisticsTokensConverter : public IStringFieldConverter void handle_index_term(vespalib::stringref word, vespalib::slime::Inserter& inserter); void handle_indexing_terms(const document::StringFieldValue& value, vespalib::slime::Inserter& inserter); public: - LinguisticsTokensConverter(const linguistics::TokenExtractor& token_extractor); - ~LinguisticsTokensConverter() override; + TokensConverter(const linguistics::TokenExtractor& token_extractor); + ~TokensConverter() override; void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; bool render_weighted_set_as_array() const override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp index 5e94e270c53..0741e5cc352 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp @@ -1,34 +1,34 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "linguistics_tokens_dfw.h" +#include "tokens_dfw.h" #include "i_docsum_store_document.h" -#include "linguistics_tokens_converter.h" +#include "tokens_converter.h" #include <vespa/searchlib/memoryindex/field_inverter.h> using search::memoryindex::FieldInverter; namespace search::docsummary { -LinguisticsTokensDFW::LinguisticsTokensDFW(const vespalib::string& input_field_name) +TokensDFW::TokensDFW(const vespalib::string& input_field_name) : DocsumFieldWriter(), _input_field_name(input_field_name), _token_extractor(_input_field_name, FieldInverter::max_word_len) { } -LinguisticsTokensDFW::~LinguisticsTokensDFW() = default; +TokensDFW::~TokensDFW() = default; bool -LinguisticsTokensDFW::isGenerated() const +TokensDFW::isGenerated() const { return false; } void -LinguisticsTokensDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter& target) const +TokensDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter& target) const { if (doc != nullptr) { - LinguisticsTokensConverter converter(_token_extractor); + TokensConverter converter(_token_extractor); doc->insert_summary_field(_input_field_name, target, &converter); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h index 9c6955b322e..e9f91ab683a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h @@ -10,17 +10,17 @@ namespace search::docsummary { /* * Class for writing annotated string field values from document as - * arrays containing the indexing terms. + * arrays containing the tokens. */ -class LinguisticsTokensDFW : public DocsumFieldWriter +class TokensDFW : public DocsumFieldWriter { private: vespalib::string _input_field_name; linguistics::TokenExtractor _token_extractor; public: - explicit LinguisticsTokensDFW(const vespalib::string& input_field_name); - ~LinguisticsTokensDFW() override; + explicit TokensDFW(const vespalib::string& input_field_name); + ~TokensDFW() override; bool isGenerated() const override; void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter& target) const override; }; |