diff options
38 files changed, 247 insertions, 167 deletions
diff --git a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java index f89e79c4cfa..f0f85fb78dd 100644 --- a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java +++ b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java @@ -62,8 +62,10 @@ public final class DocumentProcessing { @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessor.Progress process(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); - // TODO Vespa 8: Use TBD instead, this method will be removed + + // TODO Vespa 8: Remove statement (registry will be removed from Processing) processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); + return executor.processUntilDone(processing); } @@ -83,8 +85,10 @@ public final class DocumentProcessing { @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessor.Progress processOnce(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); - // TODO Vespa 8: Use TBD instead, this method will be removed + + // TODO Vespa 8: Remove statement (registry will be removed from Processing) processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); + return executor.process(processing); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index d9bdf5dc917..0d1222e737b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -9,6 +9,7 @@ import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; @@ -186,6 +187,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, ": Dense tensor dimensions must have a size"); addQueryCommand("type " + type); } + else if (dataType instanceof WeightedSetDataType) { + var nested = ((WeightedSetDataType) dataType).getNestedType().getName(); + addQueryCommand("type WeightedSet<" + nested + ">"); + } else { addQueryCommand("type " + dataType.getName()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java index e7423b17830..3cf5628d282 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -138,6 +138,7 @@ public class ConvertParsedFields { if (indexing.isPresent()) { field.setIndexingScript(indexing.get().script()); } + parsed.getWeight().ifPresent(value -> field.setWeight(value)); parsed.getStemming().ifPresent(value -> field.setStemming(value)); parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); for (var attribute : parsed.getAttributes()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 332a2153516..21a68744c19 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -239,7 +239,7 @@ public class ConvertSchemaCollection { if (parsed.hasStemming()) { schema.setStemming(parsed.getStemming()); } - schema.enableRawAsBase64(parsed.getRawAsBase64()); + parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); var typeContext = typeConverter.makeContext(parsed.getDocument()); var fieldConverter = new ConvertParsedFields(typeContext); convertDocument(schema, parsed.getDocument(), fieldConverter); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java index 5bcfa841bae..ca876997dc6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java @@ -58,6 +58,7 @@ class ParsedField extends ParsedBlock { List<String> getQueryCommands() { return List.copyOf(queryCommands); } String lookupAliasedFrom(String alias) { return aliases.get(alias); } ParsedMatchSettings matchSettings() { return this.matchInfo; } + Optional<Integer> getWeight() { return Optional.ofNullable(weight); } Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); } Optional<String> getNormalizing() { return Optional.ofNullable(normalizing); } Optional<ParsedIndexingOp> getIndexing() { return Optional.ofNullable(indexingOp); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index 0004094e1c2..599dd6e2a7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * This class holds the extracted information after parsing @@ -30,7 +31,7 @@ public class ParsedSchema extends ParsedBlock { } private boolean documentWithoutSchema = false; - private boolean rawAsBase64 = false; // TODO Vespa 8 flip default + private Boolean rawAsBase64 = null; private ParsedDocument myDocument = null; private Stemming defaultStemming = null; private final List<ImportedField> importedFields = new ArrayList<>(); @@ -53,7 +54,7 @@ public class ParsedSchema extends ParsedBlock { } boolean getDocumentWithoutSchema() { return documentWithoutSchema; } - boolean getRawAsBase64() { return rawAsBase64; } + Optional<Boolean> getRawAsBase64() { return Optional.ofNullable(rawAsBase64); } boolean hasDocument() { return myDocument != null; } ParsedDocument getDocument() { return myDocument; } boolean hasStemming() { return defaultStemming != null; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index 3aed90a58e1..d04277706a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -46,6 +46,7 @@ class ParsedType { case "raw": return Variant.BUILTIN; case "tag": return Variant.BUILTIN; case "position": return Variant.POSITION; + case "float16": return Variant.BUILTIN; } return Variant.UNKNOWN; } diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index 2db4ead180b..7f43cd67a6b 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -96,7 +96,7 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "tagfield" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "tagfield" -indexinfo[].command[].command "type tag" +indexinfo[].command[].command "type WeightedSet<string>" indexinfo[].command[].indexname "structfield.s1" indexinfo[].command[].command "index" indexinfo[].command[].indexname "structfield.s1" @@ -704,4 +704,4 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "pst_sta_boldingoff_nomatch_tag_01" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "pst_sta_boldingoff_nomatch_tag_01" -indexinfo[].command[].command "type tag" +indexinfo[].command[].command "type WeightedSet<string>" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java index 231881114b5..32f82f44ad0 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.searchdefinition.derived.Deriver; import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; import static org.junit.Assert.fail; @@ -36,9 +37,12 @@ public class StructTestCase extends AbstractSchemaTestCase { } @Test + @Ignore public void testStructAndDocumentWithSameNames() { try { DocumenttypesConfig.Builder dt = Deriver.getDocumentTypesConfig("src/test/examples/structanddocumentwithsamenames.sd"); + // while the above line may work, the config generated will fail. + // See also NameCollisionTestCase. } catch (Exception e) { fail("Should not have thrown exception " + e); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java index 5fce5c06943..70f6187be12 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java @@ -2,7 +2,12 @@ package com.yahoo.searchdefinition.derived; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.searchdefinition.ApplicationBuilder; + import org.junit.Test; +import static org.junit.Assert.assertThrows; /** * Verifies that a struct in a document type is preferred over another document type @@ -14,7 +19,21 @@ public class NameCollisionTestCase extends AbstractExportingTestCase { @Test public void testNameCollision() throws Exception { - assertCorrectDeriving("namecollision", "collisionstruct", new TestableDeployLogger()); + var ex = assertThrows(IllegalArgumentException.class, () -> { + assertCorrectDeriving("namecollision", "collisionstruct", + new TestProperties().setExperimentalSdParsing(false), + new TestableDeployLogger()); + var docman = DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); + }); + System.err.println("MSG 1: "+ex.getClass()+" -> "+ex.getMessage()); + var ey = assertThrows(IllegalArgumentException.class, () -> { + assertCorrectDeriving("namecollision", "collisionstruct", + new TestProperties().setExperimentalSdParsing(true), + new TestableDeployLogger()); + var docman = DocumentTypeManager.fromFile("temp/namecollision/documentmanager.cfg"); + }); + System.err.println("MSG 2: "+ey.getMessage()); + } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java index 190a8fc0a19..8bd04af8c54 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java @@ -256,7 +256,6 @@ public class IntermediateParserTestCase { checkFileParses("src/test/examples/stemmingdefault.sd"); checkFileParses("src/test/examples/stemmingsetting.sd"); checkFileParses("src/test/examples/strange.sd"); - checkFileParses("src/test/examples/structanddocumentwithsamenames.sd"); checkFileParses("src/test/examples/struct.sd"); checkFileParses("src/test/examples/summaryfieldcollision.sd"); checkFileParses("src/test/examples/weightedset-summaryto.sd"); diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java index 97ae9e8b113..b89994d63da 100644 --- a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java @@ -6,7 +6,7 @@ import java.util.Set; /** * A hit which holds information on error conditions in a result. - * En error hit maintains a main error - the main error of the result. + * An error hit maintains a main error - the main error of the result. * * @author bratseth */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 4ab7901cd62..30f416747e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -215,6 +215,7 @@ public class Application { public Optional<ApplicationVersion> oldestDeployedApplication() { return productionDeployments().values().stream().flatMap(List::stream) .map(Deployment::applicationVersion) + .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) .min(Comparator.naturalOrder()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 88df49ad3ab..5c3c43461dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -446,21 +446,17 @@ public class ApplicationController { controller.notificationsDb().removeNotifications(notification.source()); } - var oldestDeployedVersion = application.get().productionDeployments().values().stream() - .flatMap(List::stream) - .map(Deployment::applicationVersion) - .filter(version -> ! version.isDeployedDirectly()) - .min(naturalOrder()) - .orElse(ApplicationVersion.unknown); - - var olderVersions = application.get().versions().stream() - .filter(version -> version.compareTo(oldestDeployedVersion) < 0) - .sorted() - .collect(Collectors.toList()); + ApplicationVersion oldestDeployedVersion = application.get().oldestDeployedApplication() + .orElse(ApplicationVersion.unknown); + + List<ApplicationVersion> olderVersions = application.get().versions().stream() + .filter(version -> version.compareTo(oldestDeployedVersion) < 0) + .sorted() + .collect(Collectors.toList()); // Remove any version not deployed anywhere - but keep one - for (int i = 0; i < olderVersions.size() - 1; i++) { - application = application.withoutVersion(olderVersions.get(i)); + for (ApplicationVersion version : olderVersions) { + application = application.withoutVersion(version); } store(application); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 8c933f98277..09fd9ca86bc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -43,7 +43,6 @@ import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static java.util.Collections.reverseOrder; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; @@ -201,7 +200,7 @@ public class DeploymentStatus { jobs.putAll(productionJobs); // Add runs for idle, declared test jobs if they have no successes on their instance's change's versions. jobSteps.forEach((job, step) -> { - if ( ! step.isDeclared() || step.type() != StepType.test || jobs.containsKey(job)) + if ( ! step.isDeclared() || job.type().isProduction() || jobs.containsKey(job)) return; Change change = changes.get(job.application().instance()); @@ -212,9 +211,8 @@ public class DeploymentStatus { .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment()) .filter(jobId -> deploymentFor(jobId).isPresent()) .findFirst(); - Versions versions = Versions.from(change, application, firstProductionJobWithDeployment.flatMap(this::deploymentFor), systemVersion); - if (step.completedAt(change, firstProductionJobWithDeployment).isEmpty()) + if (step.completedAt(change, Optional.empty()).isEmpty()) jobs.merge(job, List.of(new Job(job.type(), versions, step.readyAt(change), change)), DeploymentStatus::union); }); return Collections.unmodifiableMap(jobs); @@ -266,16 +264,18 @@ public class DeploymentStatus { * For the "exclusive" revision upgrade policy it is the oldest such revision; otherwise, it is the latest. */ public Change outstandingChange(InstanceName instance) { - return Optional.ofNullable(instanceSteps().get(instance)) - .flatMap(instanceStatus -> application.deployableVersions(application.deploymentSpec().requireInstance(instance).revisionTarget() == next).stream() - .filter(version -> instanceStatus.dependenciesCompletedAt(Change.of(version), Optional.empty()).map(at -> ! at.isAfter(now)).orElse(false)) - .filter(version -> application.productionDeployments().getOrDefault(instance, List.of()).stream() - .noneMatch(deployment -> deployment.applicationVersion().compareTo(version) > 0)) - .map(Change::of) - .filter(change -> application.require(instance).change().application().map(change::upgrades).orElse(true)) - .filter(change -> ! hasCompleted(instance, change)) - .findFirst()) - .orElse(Change.empty()); + StepStatus status = instanceSteps().get(instance); + if (status == null) return Change.empty(); + for (ApplicationVersion version : application.deployableVersions(application.deploymentSpec().requireInstance(instance).revisionTarget() == next)) { + if (status.dependenciesCompletedAt(Change.of(version), Optional.empty()).map(now::isBefore).orElse(true)) continue; + Change change = Change.of(version); + if (application.productionDeployments().getOrDefault(instance, List.of()).stream() + .anyMatch(deployment -> change.downgrades(deployment.applicationVersion()))) continue; + if ( ! application.require(instance).change().application().map(change::upgrades).orElse(true)) continue; + if (hasCompleted(instance, change)) continue; + return change; + } + return Change.empty(); } /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ @@ -805,7 +805,7 @@ public class DeploymentStatus { .orElse(false)) return job.lastCompleted().flatMap(Run::end); - return (dependent.equals(job()) ? job.lastSuccess().stream() + return (dependent.equals(job()) ? job.lastTriggered().filter(run -> run.status() == RunStatus.success).stream() : RunList.from(job).status(RunStatus.success).asList().stream()) .filter(run -> change.platform().map(run.versions().targetPlatform()::equals).orElse(true) && change.application().map(run.versions().targetApplication()::equals).orElse(true)) @@ -857,10 +857,13 @@ public class DeploymentStatus { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { return RunList.from(job) - .matching(run -> run.versions().targetsMatch(Versions.from(change, - status.application, - dependent.flatMap(status::deploymentFor), - status.systemVersion))) + .matching(run -> dependent.flatMap(status::deploymentFor) + .map(deployment -> run.versions().targetsMatch(Versions.from(change, + status.application, + Optional.of(deployment), + status.systemVersion))) + .orElseGet(() -> (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform())) + && (change.application().isEmpty() || change.application().get().equals(run.versions().targetApplication())))) .status(RunStatus.success) .asList().stream() .map(run -> run.end().get()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 66a30050f7a..647b52f7d51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -607,11 +607,7 @@ public class JobController { private void prunePackages(TenantAndApplicationId id) { controller.applications().lockApplicationIfPresent(id, application -> { - application.get().productionDeployments().values().stream() - .flatMap(List::stream) - .map(Deployment::applicationVersion) - .filter(version -> ! version.isUnknown() && ! version.isDeployedDirectly()) - .min(naturalOrder()) + application.get().oldestDeployedApplication() .ifPresent(oldestDeployed -> { controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); controller.applications().applicationStore().pruneTesters(id.tenant(), id.application(), oldestDeployed); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 21cc69369d8..989a7c31821 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -236,8 +236,13 @@ public class DeploymentContext { /** Flush count pending DNS updates */ public DeploymentContext flushDnsUpdates(int count) { var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofSeconds(count)); - dispatcher.run(); - return this; + try { + dispatcher.run(); + return this; + } + finally { + dispatcher.shutdown(); + } } /** Add a routing policy for this in given zone, with status set to inactive */ @@ -255,6 +260,11 @@ public class DeploymentContext { } /** Submit given application package for deployment */ + public DeploymentContext resubmit(ApplicationPackage applicationPackage) { + return submit(applicationPackage, Optional.of(defaultSourceRevision), salt.get()); + } + + /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { return submit(applicationPackage, Optional.of(defaultSourceRevision)); } @@ -266,7 +276,7 @@ public class DeploymentContext { /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) { - return submit(applicationPackage, sourceRevision, salt.getAndIncrement()); + return submit(applicationPackage, sourceRevision, salt.incrementAndGet()); } /** Submit given application package for deployment */ @@ -597,8 +607,9 @@ public class DeploymentContext { runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); assertFalse(jobs.run(id).get().hasFailed()); - assertEquals(job.type().isProduction(), instance().deployments().containsKey(zone)); - assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(id.application()).id())).isEmpty()); + Instance instance = tester.application(TenantAndApplicationId.from(instanceId)).require(id.application().instance()); + assertEquals(job.type().isProduction(), instance.deployments().containsKey(zone)); + assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(instance.id()).id())).isEmpty()); } private JobId jobId(JobType type) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index b2847a29654..c8126207a73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1873,7 +1873,7 @@ public class DeploymentTriggerTest { // Deploy manually again, then submit new package. app.runJob(productionCdUsEast1, cdPackage); app.submit(cdPackage); - app.runJob(systemTest); + app.triggerJobs().jobAborted(systemTest).runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); app.runJob(productionCdUsEast1) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 265dedec8d0..78341682f75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -945,9 +945,9 @@ public class UpgraderTest { // Application downgrades to pinned version. tester.abortAll(); - context.runJob(stagingTest).runJob(productionUsCentral1); + context.runJob(stagingTest).runJob(productionUsCentral1).runJob(productionUsWest1); assertTrue(context.instance().change().hasTargets()); - context.runJob(productionUsWest1); // us-east-3 never upgraded, so no downgrade is needed. + context.runJob(productionUsEast3); assertFalse(context.instance().change().hasTargets()); } @@ -1011,7 +1011,7 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(v2); tester.upgrader().maintain(); assertEquals(Change.of(v2), application.instance().change()); - application.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1); + application.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1).timeOutConvergence(productionUsWest1); tester.triggerJobs(); // While second deployment completes upgrade, version confidence becomes broken and upgrade is cancelled diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index 47ebb6a7988..834d63c5a86 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -140,10 +140,8 @@ public class Processing { } /** - * @deprecated Use TBD instead + * @deprecated This method will be removed without replacement in Vespa 8. */ - // TODO: used to: processing.setDocprocServiceRegistry(this.documentProcessingHandler.getDocprocServiceRegistry()); - // from Processor and LoggingRequestHandler @Deprecated(forRemoval = true, since="7") @SuppressWarnings("removal") // TODO Vespa 8: remove public void setDocprocServiceRegistry(ComponentRegistry<DocprocService> docprocServiceRegistry) { @@ -166,9 +164,9 @@ public class Processing { * if #getServiceName returns a name that is not registered in {@link com.yahoo.docproc.DocprocService}. * * @return the service processing this, or null if unknown. - * @deprecated Use TBD instead + * @deprecated Formerly used to retrieve the {@link com.yahoo.document.DocumentTypeManager}, + * which can now be directly injected via your component constructor. */ - // TODO: used to getService().getDocumentTypeManager() in subclasses of DocumentProcessor @Deprecated(forRemoval = true, since="7") @SuppressWarnings("removal") // TODO Vespa 8: remove public DocprocService getService() { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index e8a2e214776..33cd6647ede 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -105,8 +105,10 @@ class ProcessingFactory { Processing processing = new Processing(); processing.addDocumentOperation(documentOperation); processing.setServiceName(serviceName); - // TODO Vespa 8: Use TBD instead, this method will be removed + + // TODO Vespa 8: Remove statement (registry will be removed from Processing) processing.setDocprocServiceRegistry(docprocServiceComponentRegistry); + processing.setVariable("route", message.getRoute()); processing.setVariable("timeout", message.getTimeRemaining()); return processing; diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index bb1777954a3..697536a8ac0 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -52,6 +52,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static DocumentType DOCUMENT = new DocumentType("document"); public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); + final static int TAG_ID = 18; public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index be2c182426e..9325e374daa 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -61,8 +61,13 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub public ConfigSubscriber configure(String configId) { ConfigSubscriber subscriber = new ConfigSubscriber(); - subscriber.subscribe(this, DocumentmanagerConfig.class, configId); - return subscriber; + try { + subscriber.subscribe(this, DocumentmanagerConfig.class, configId); + return subscriber; + } catch (RuntimeException e) { + subscriber.close(); + throw e; + } } /** One-shot configuration; should be called on a newly constructed manager */ @@ -264,7 +269,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub var old = configMap.put(id, dataTypeConfig); if (old != null) { throw new IllegalArgumentException - ("Multiple configs for id "+id+" first: "+old+" second: "+dataTypeConfig); + ("Multiple configs for id "+id+" first:\n"+old+"\nsecond:\n"+dataTypeConfig); } } } diff --git a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java index 35dd13efb0b..b21f059bd7d 100644 --- a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java +++ b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java @@ -24,25 +24,33 @@ public class WeightedSetDataType extends CollectionDataType { public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { this(nestedType, createIfNonExistent, removeIfZero, 0); - if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { // the tag type definition - setId(18); - } else { - setId(getName().toLowerCase().hashCode()); - } } public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero, int id) { super(createName(nestedType, createIfNonExistent, removeIfZero), id, nestedType); this.createIfNonExistent = createIfNonExistent; this.removeIfZero = removeIfZero; + if (id == 0) { + if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { // the tag type definition + setId(TAG_ID); + } else { + setId(getName().toLowerCase().hashCode()); + } + } + int code = getId(); + if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != TAG_ID)) { + throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); + } } + /* + * @deprecated // TODO remove on Vespa 8 + * Do not use - use one of the constructors above. + * Note: ignores typeName argument. + */ + @Deprecated public WeightedSetDataType(String typeName, int code, DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { - super(typeName != null ? createName(nestedType, createIfNonExistent, removeIfZero) : null, code, nestedType); - if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != 18)) // 18 == DataType.TAG.getId() is not yet initialized - throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); - this.createIfNonExistent = createIfNonExistent; - this.removeIfZero = removeIfZero; + this(nestedType, createIfNonExistent, removeIfZero, code); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index ba8aa49d87c..338187c5270 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -122,6 +123,10 @@ public class NodeRepoStats { public static class ApplicationStats implements Comparable<ApplicationStats> { + private static final Comparator<ApplicationStats> comparator = Comparator.comparingDouble(ApplicationStats::unutilizedCost).reversed() + .thenComparingDouble(ApplicationStats::cost) + .thenComparing(ApplicationStats::id); + private final ApplicationId id; private final Load load; private final double cost; @@ -148,7 +153,7 @@ public class NodeRepoStats { @Override public int compareTo(NodeRepoStats.ApplicationStats other) { - return -Double.compare(this.unutilizedCost(), other.unutilizedCost()); + return comparator.compare(this, other); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json index 8867520fef6..8a46f8115be 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json @@ -11,7 +11,7 @@ }, "applications": [ { - "id": "tenant3.application3.instance3", + "id": "tenant1.application1.instance1", "load": { "cpu": 0.0, "memory": 0.0, @@ -31,7 +31,7 @@ "unutilizedCost": 0.0 }, { - "id": "tenant4.application4.instance4", + "id": "tenant3.application3.instance3", "load": { "cpu": 0.0, "memory": 0.0, @@ -41,7 +41,7 @@ "unutilizedCost": 0.0 }, { - "id": "tenant1.application1.instance1", + "id": "tenant4.application4.instance4", "load": { "cpu": 0.0, "memory": 0.0, diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchcommon/src/vespa/searchcommon/attribute/status.cpp index 0509e9a51c4..a7d1f5b3d38 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/status.cpp @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "status.h" +#include <vespa/vespalib/util/atomic.h> + +using namespace vespalib::atomic; + namespace search::attribute { Status::Status() @@ -21,15 +25,15 @@ Status::Status() } Status::Status(const Status& rhs) - : _numDocs(rhs._numDocs), - _numValues(rhs._numValues), - _numUniqueValues(rhs._numUniqueValues), - _allocated(rhs._allocated), - _used(rhs._used), - _dead(rhs._dead), - _unused(rhs._unused), - _onHold(rhs._onHold), - _onHoldMax(rhs._onHoldMax), + : _numDocs(load_relaxed(rhs._numDocs)), + _numValues(load_relaxed(rhs._numValues)), + _numUniqueValues(load_relaxed(rhs._numUniqueValues)), + _allocated(load_relaxed(rhs._allocated)), + _used(load_relaxed(rhs._used)), + _dead(load_relaxed(rhs._dead)), + _unused(load_relaxed(rhs._unused)), + _onHold(load_relaxed(rhs._onHold)), + _onHoldMax(load_relaxed(rhs._onHoldMax)), _lastSyncToken(rhs.getLastSyncToken()), _updates(rhs._updates), _nonIdempotentUpdates(rhs._nonIdempotentUpdates), @@ -40,15 +44,15 @@ Status::Status(const Status& rhs) Status& Status::operator=(const Status& rhs) { - _numDocs = rhs._numDocs; - _numValues = rhs._numValues; - _numUniqueValues = rhs._numUniqueValues; - _allocated = rhs._allocated; - _used = rhs._used; - _dead = rhs._dead; - _unused = rhs._unused; - _onHold = rhs._onHold; - _onHoldMax = rhs._onHoldMax; + store_relaxed(_numDocs, load_relaxed(rhs._numDocs)); + store_relaxed(_numValues, load_relaxed(rhs._numValues)); + store_relaxed(_numUniqueValues, load_relaxed(rhs._numUniqueValues)); + store_relaxed(_allocated, load_relaxed(rhs._allocated)); + store_relaxed(_used, load_relaxed(rhs._used)); + store_relaxed(_dead, load_relaxed(rhs._dead)); + store_relaxed(_unused, load_relaxed(rhs._unused)); + store_relaxed(_onHold, load_relaxed(rhs._onHold)); + store_relaxed(_onHoldMax, load_relaxed(rhs._onHoldMax)); setLastSyncToken(rhs.getLastSyncToken()); _updates = rhs._updates; _nonIdempotentUpdates = rhs._nonIdempotentUpdates; @@ -69,14 +73,14 @@ void Status::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold) { - _numValues = numValues; - _numUniqueValues = numUniqueValue; - _allocated = allocated; - _used = used; - _dead = dead; - _unused = allocated - used; - _onHold = onHold; - _onHoldMax = std::max(_onHoldMax, onHold); + store_relaxed(_numValues, numValues); + store_relaxed(_numUniqueValues, numUniqueValue); + store_relaxed(_allocated, allocated); + store_relaxed(_used, used); + store_relaxed(_dead, dead); + store_relaxed(_unused, allocated - used); + store_relaxed(_onHold, onHold); + store_relaxed(_onHoldMax, std::max(load_relaxed(_onHoldMax), onHold)); } } diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchcommon/src/vespa/searchcommon/attribute/status.h index 0280bfaae4e..f2212d4c76a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.h +++ b/searchcommon/src/vespa/searchcommon/attribute/status.h @@ -17,22 +17,23 @@ public: void updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold); - uint64_t getNumDocs() const { return _numDocs; } - uint64_t getNumValues() const { return _numValues; } - uint64_t getNumUniqueValues() const { return _numUniqueValues; } - uint64_t getAllocated() const { return _allocated; } - uint64_t getUsed() const { return _used; } - uint64_t getDead() const { return _dead; } - uint64_t getOnHold() const { return _onHold; } - uint64_t getOnHoldMax() const { return _onHoldMax; } + uint64_t getNumDocs() const { return _numDocs.load(std::memory_order_relaxed); } + uint64_t getNumValues() const { return _numValues.load(std::memory_order_relaxed); } + uint64_t getNumUniqueValues() const { return _numUniqueValues.load(std::memory_order_relaxed); } + uint64_t getAllocated() const { return _allocated.load(std::memory_order_relaxed); } + uint64_t getUsed() const { return _used.load(std::memory_order_relaxed); } + uint64_t getDead() const { return _dead.load(std::memory_order_relaxed); } + uint64_t getOnHold() const { return _onHold.load(std::memory_order_relaxed); } + uint64_t getOnHoldMax() const { return _onHoldMax.load(std::memory_order_relaxed); } // This might be accessed from other threads than the writer thread. uint64_t getLastSyncToken() const { return _lastSyncToken.load(std::memory_order_relaxed); } uint64_t getUpdateCount() const { return _updates; } uint64_t getNonIdempotentUpdateCount() const { return _nonIdempotentUpdates; } uint32_t getBitVectors() const { return _bitVectors; } - void setNumDocs(uint64_t v) { _numDocs = v; } - void incNumDocs() { ++_numDocs; } + void setNumDocs(uint64_t v) { _numDocs.store(v, std::memory_order_relaxed); } + void incNumDocs() { _numDocs.store(_numDocs.load(std::memory_order_relaxed) + 1u, + std::memory_order_relaxed); } void setLastSyncToken(uint64_t v) { _lastSyncToken.store(v, std::memory_order_relaxed); } void incUpdates(uint64_t v=1) { _updates += v; } void incNonIdempotentUpdates(uint64_t v = 1) { _nonIdempotentUpdates += v; } @@ -42,15 +43,15 @@ public: static vespalib::string createName(vespalib::stringref index, vespalib::stringref attr); private: - uint64_t _numDocs; - uint64_t _numValues; - uint64_t _numUniqueValues; - uint64_t _allocated; - uint64_t _used; - uint64_t _dead; - uint64_t _unused; - uint64_t _onHold; - uint64_t _onHoldMax; + std::atomic<uint64_t> _numDocs; + std::atomic<uint64_t> _numValues; + std::atomic<uint64_t> _numUniqueValues; + std::atomic<uint64_t> _allocated; + std::atomic<uint64_t> _used; + std::atomic<uint64_t> _dead; + std::atomic<uint64_t> _unused; + std::atomic<uint64_t> _onHold; + std::atomic<uint64_t> _onHoldMax; std::atomic<uint64_t> _lastSyncToken; uint64_t _updates; uint64_t _nonIdempotentUpdates; diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 8489a68af15..0ec27ac419f 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -177,14 +177,14 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req) } bool MatchEngine::isOnline() const { - return _nodeUp; + return _nodeUp.load(std::memory_order_relaxed); } void MatchEngine::setNodeUp(bool nodeUp) { - _nodeUp = nodeUp; + _nodeUp.store(nodeUp, std::memory_order_relaxed); } void @@ -192,7 +192,7 @@ MatchEngine::setNodeMaintenance(bool nodeMaintenance) { _nodeMaintenance = nodeMaintenance; if (nodeMaintenance) { - _nodeUp = false; + _nodeUp.store(false, std::memory_order_relaxed); } } diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 3d3be775a4a..fcafdc5a5f8 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -25,7 +25,7 @@ private: HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; vespalib::SimpleThreadBundle::Pool _threadBundlePool; - bool _nodeUp; + std::atomic<bool> _nodeUp; bool _nodeMaintenance; public: diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index f740cec7900..20aba93d4d7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -9,7 +9,7 @@ #include "indexreadutilities.h" #include "indexwriteutilities.h" #include "index_disk_dir.h" -#include <vespa/document/fieldvalue//document.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchcorespi/flush/lambdaflushtask.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index e6e7d777656..bf437dd7ee3 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -242,7 +242,7 @@ WarmupIndexCollection::WarmupTask::WarmupTask(std::unique_ptr<MatchData> md, std _retainGuard(_warmup->_pendingTasks), _matchData(std::move(md)), _bluePrint(), - _requestContext(warmup->_clock) + _requestContext(_warmup->_clock) { } diff --git a/searchlib/src/vespa/searchlib/attribute/atomic_utils.h b/searchlib/src/vespa/searchlib/attribute/atomic_utils.h new file mode 100644 index 00000000000..48914de8942 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/atomic_utils.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::datastore { + +class AtomicEntryRef; +class EntryRef; + +} + +namespace search::attribute::atomic_utils { + +/* + * Helper class to map from atomic value to non-atomic value, e.g. + * from AtomicEntryRef to EntryRef. + */ +template <typename MaybeAtomicValue> +class NonAtomicValue { +public: + using type = MaybeAtomicValue; +}; + +template <> +class NonAtomicValue<vespalib::datastore::AtomicEntryRef> +{ +public: + using type = vespalib::datastore::EntryRef; +}; + +template <class MaybeAtomicValue> +using NonAtomicValue_t = typename NonAtomicValue<MaybeAtomicValue>::type; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index 2c53003112a..707cdc6af76 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -85,12 +85,12 @@ template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType #define INSTANTIATE_WSET(ValueType, Saver) \ template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<WeightedValue<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE(ValueType, Saver) \ -template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<load_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) +template void loadFromEnumeratedSingleValue(vespalib::RcuVectorBase<ValueType> &, vespalib::GenerationHolder &, ReaderBase &, vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<ValueType>>, vespalib::ConstArrayRef<uint32_t>, Saver) #define INSTANTIATE_SINGLE_ARRAY_WSET(ValueType, Saver) \ INSTANTIATE_SINGLE(ValueType, Saver); \ -INSTANTIATE_ARRAY(load_utils::NonAtomicValue_t<ValueType>, Saver); \ -INSTANTIATE_WSET(load_utils::NonAtomicValue_t<ValueType>, Saver) +INSTANTIATE_ARRAY(atomic_utils::NonAtomicValue_t<ValueType>, Saver); \ +INSTANTIATE_WSET(atomic_utils::NonAtomicValue_t<ValueType>, Saver) #define INSTANTIATE_ENUM(Saver) \ INSTANTIATE_SINGLE_ARRAY_WSET(AtomicEntryRef, Saver) diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index fe41811dcfa..f9f933f3726 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -2,6 +2,7 @@ #pragma once +#include "atomic_utils.h" #include "attributevector.h" #include "readerbase.h" #include <vespa/vespalib/util/arrayref.h> @@ -15,30 +16,6 @@ class EntryRef; namespace search::attribute { -namespace load_utils { - -/* - * Helper class to map from atomic value to non-atomic value, e.g. - * from AtomicEntryRef to EntryRef. - */ -template <typename MaybeAtomicValue> -class NonAtomicValue { -public: - using type = MaybeAtomicValue; -}; - -template <> -class NonAtomicValue<vespalib::datastore::AtomicEntryRef> -{ -public: - using type = vespalib::datastore::EntryRef; -}; - -template <class MaybeAtomicValue> -using NonAtomicValue_t = typename NonAtomicValue<MaybeAtomicValue>::type; - -} - /** * Helper functions used to open / load attribute vector data files from disk. */ @@ -82,7 +59,7 @@ void loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, - vespalib::ConstArrayRef<load_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) __attribute((noinline)); diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 92cbc72ae2c..74126299919 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -54,12 +54,12 @@ void loadFromEnumeratedSingleValue(Vector &vector, vespalib::GenerationHolder &genHolder, ReaderBase &attrReader, - vespalib::ConstArrayRef<load_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, + vespalib::ConstArrayRef<atomic_utils::NonAtomicValue_t<typename Vector::ValueType>> enumValueToValueMap, vespalib::ConstArrayRef<uint32_t> enum_value_remapping, Saver saver) { using ValueType = typename Vector::ValueType; - using NonAtomicValueType = load_utils::NonAtomicValue_t<ValueType>; + using NonAtomicValueType = atomic_utils::NonAtomicValue_t<ValueType>; uint32_t numDocs = attrReader.getEnumCount(); genHolder.clearHoldLists(); vector.reset(); diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index 2c93312eb6d..f035e2c6f00 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -8,6 +8,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -99,7 +100,7 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte /** Returns the items grouped by the given classifier. */ public final <OtherType> Map<OtherType, ListType> groupingBy(Function<Type, OtherType> classifier) { return items.stream().collect(Collectors.groupingBy(classifier, - HashMap::new, + LinkedHashMap::new, Collectors.collectingAndThen(toUnmodifiableList(), (list) -> constructor.apply(list, false)))); } diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp index 7d11990577d..4716e91c2c4 100644 --- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp +++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp @@ -196,9 +196,11 @@ protected: using MyTree = typename Params::MyTree; using MyTreeIterator = typename MyTree::Iterator; using MyTreeConstIterator = typename MyTree::ConstIterator; + using KeyStore = IntStore; + using ValueStore = IntStore; GenerationHandler _generationHandler; - IntStore _keys; - IntStore _values; + KeyStore _keys; + ValueStore _values; MyTree _tree; MyTreeIterator _writeItr; vespalib::ThreadStackExecutor _writer; // 1 write thread @@ -344,7 +346,7 @@ template <typename Params> void Fixture<Params>::compact_keys() { - if constexpr (_keys.is_indirect) { + if constexpr (KeyStore::is_indirect) { auto to_hold = _keys.start_compact(); EntryRefFilter filter(_keys.get_num_buffers(), _keys.get_offset_bits()); filter.add_buffers(to_hold); @@ -366,7 +368,7 @@ template <typename Params> void Fixture<Params>::compact_values() { - if constexpr (_values.is_indirect) { + if constexpr (ValueStore::is_indirect) { auto to_hold = _values.start_compact(); EntryRefFilter filter(_values.get_num_buffers(), _values.get_offset_bits()); filter.add_buffers(to_hold); @@ -391,12 +393,12 @@ Fixture<Params>::consider_compact(uint32_t idx) if (_compact_tree.consider(idx) && !_tree.getAllocator().getNodeStore().has_held_buffers()) { compact_tree(); } - if constexpr (_keys.is_indirect) { + if constexpr (KeyStore::is_indirect) { if (_compact_keys.consider(idx) && !_keys.has_held_buffers()) { compact_keys(); } } - if constexpr (_values.is_indirect) { + if constexpr (ValueStore::is_indirect) { if (_compact_values.consider(idx) && !_values.has_held_buffers()) { compact_values(); } |