diff options
119 files changed, 1373 insertions, 2314 deletions
diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 6e9ae6463fb..5e5325fd5ad 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -187,8 +187,8 @@ <include>junit:junit:4.13.2:test</include> <include>net.java.dev.jna:jna:5.11.0:test</include> <include>net.openhft:zero-allocation-hashing:jar:0.16:test</include> - <include>org.antlr:antlr-runtime:3.5.2:test</include> - <include>org.antlr:antlr4-runtime:4.9.3:test</include> + <include>org.antlr:antlr-runtime:3.5.3:test</include> + <include>org.antlr:antlr4-runtime:4.11.1:test</include> <include>org.apache.commons:commons-exec:1.3:test</include> <include>org.apache.commons:commons-math3:3.6.1:test</include> <include>org.apache.commons:commons-compress:jar:1.22:test</include> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 5e740c5f03c..7d352373e2a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +// TODO: RpcServer is only used in unit tests, should be removed public class RpcServer { private static final Logger log = Logger.getLogger(RpcServer.class.getName()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index e0f6546e410..0819e9981ab 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -10,7 +10,6 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; -import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.ClusterState; @@ -18,7 +17,6 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; import com.yahoo.vespa.clustercontroller.core.testutils.WaitCondition; import com.yahoo.vespa.clustercontroller.core.testutils.WaitTask; @@ -40,6 +38,9 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /** + * + * Note: RpcServer is only used in unit tests + * * @author humbe */ @ExtendWith(CleanupZookeeperLogsOnSuccess.class) @@ -60,25 +61,6 @@ public class RpcServerTest extends FleetControllerTest { } @Test - void testRebinding() throws Exception { - startingTest("RpcServerTest::testRebinding"); - Slobrok slobrok = new Slobrok(); - String[] slobrokConnectionSpecs = getSlobrokConnectionSpecs(slobrok); - RpcServer server = new RpcServer(new Object(), "mycluster", 0, new BackOff()); - server.setSlobrokConnectionSpecs(slobrokConnectionSpecs, 0); - int portUsed = server.getPort(); - server.setSlobrokConnectionSpecs(slobrokConnectionSpecs, portUsed); - server.disconnect(); - server.disconnect(); - server.connect(); - server.connect(); - server.disconnect(); - server.connect(); - server.shutdown(); - slobrok.stop(); - } - - @Test void testGetSystemState() throws Exception { LogFormatter.initializeLogging(); startingTest("RpcServerTest::testGetSystemState"); diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml index 8b2e4ab484f..381a4c88946 100644 --- a/clustercontroller-utils/pom.xml +++ b/clustercontroller-utils/pom.xml @@ -16,7 +16,6 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 251bfae07bf..dc2c8d91db1 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -10,7 +10,7 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-application-package</artifactId> - <packaging>jar</packaging> + <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <dependencies> <dependency> @@ -97,7 +97,6 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -114,6 +113,11 @@ <build> <plugins> <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> @@ -135,6 +139,24 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java index b4be99ad20b..e2e2f317cee 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationId.java @@ -13,9 +13,9 @@ public enum ValidationId { indexingChange("indexing-change"), // Changing what tokens are expected and stored in field indexes indexModeChange("indexing-mode-change"), // Changing the index mode (streaming, indexed, store-only) of documents fieldTypeChange("field-type-change"), // Field type changes - clusterSizeReduction("cluster-size-reduction"), // Large reductions in cluster size + clusterSizeReduction("cluster-size-reduction"), // NOT USED. TODO: Remove on Vespa 9 tensorTypeChange("tensor-type-change"), // Tensor type change - resourcesReduction("resources-reduction"), // Large reductions in node resources (> 50% of the current min resources) + resourcesReduction("resources-reduction"), // Large reductions in node resources (> 50% of the current max total resources) contentTypeRemoval("schema-removal"), // Removal of a schema (causes deletion of all documents) contentClusterRemoval("content-cluster-removal"), // Removal (or id change) of content clusters deploymentRemoval("deployment-removal"), // Removal of production zones from deployment.xml diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 8bca56a949c..34034f5e53b 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -192,8 +192,6 @@ <!-- 3rd party artifacts embedded --> <i>aopalliance:aopalliance:*:*</i> - <i>biz.aQute.bnd:biz.aQute.bnd.util:*:*</i> - <i>biz.aQute.bnd:biz.aQute.bndlib:*:*</i> <i>com.google.errorprone:error_prone_annotations:*:*</i> <i>com.google.guava:failureaccess:*:*</i> <i>com.google.guava:guava:*:*</i> diff --git a/config-model/pom.xml b/config-model/pom.xml index e5ccfbcc51b..79e892903fa 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -289,22 +289,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>biz.aQute.bnd</groupId> - <artifactId>biz.aQute.bndlib</artifactId> - <version>6.1.0</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <exclusion> - <!-- These are not needed for our use of bndlib --> - <groupId>org.osgi</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.lz4</groupId> <artifactId>lz4-java</artifactId> </dependency> diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 16de9d3405f..9e48510e704 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -38,7 +38,9 @@ import java.util.stream.IntStream; */ public class InMemoryProvisioner implements HostProvisioner { - public static final NodeResources defaultResources = new NodeResources(1, 3, 50, 1); + public static final NodeResources defaultHostResources = new NodeResources(1, 3, 50, 1); + + private final NodeResources defaultNodeResources; /** * If this is true an exception is thrown when all nodes are used. @@ -74,32 +76,37 @@ public class InMemoryProvisioner implements HostProvisioner { /** Creates this with a number of nodes with resources 1, 3, 9, 1 */ public InMemoryProvisioner(int nodeCount, boolean sharedHosts) { - this(nodeCount, defaultResources, sharedHosts); + this(nodeCount, defaultHostResources, sharedHosts); } /** Creates this with a number of nodes with given resources */ public InMemoryProvisioner(int nodeCount, NodeResources resources, boolean sharedHosts) { - this(Map.of(resources, createHostInstances(nodeCount)), true, false, false, sharedHosts, 0); + this(Map.of(resources, createHostInstances(nodeCount)), true, false, false, sharedHosts, NodeResources.unspecified(), 0); + } + + /** Creates this with a number of nodes with given resources */ + public InMemoryProvisioner(int nodeCount, NodeResources resources, boolean sharedHosts, NodeResources defaultResources) { + this(Map.of(resources, createHostInstances(nodeCount)), true, false, false, sharedHosts, defaultResources, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, boolean sharedHosts, String... hosts) { - this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, false, sharedHosts, 0); + this(Map.of(defaultHostResources, toHostInstances(hosts)), failOnOutOfCapacity, false, false, sharedHosts, defaultHostResources, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, boolean sharedHosts, List<String> hosts) { - this(Map.of(defaultResources, toHostInstances(hosts.toArray(new String[0]))), failOnOutOfCapacity, false, false, sharedHosts, 0); + this(Map.of(defaultHostResources, toHostInstances(hosts.toArray(new String[0]))), failOnOutOfCapacity, false, false, sharedHosts, defaultHostResources, 0); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, boolean sharedHosts, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, false, sharedHosts, 0, retiredHostNames); + this(Map.of(defaultHostResources, hosts.asCollection()), failOnOutOfCapacity, false, false, sharedHosts, defaultHostResources, 0, retiredHostNames); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, boolean sharedHosts, int startIndexForClusters, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, false, sharedHosts, startIndexForClusters, retiredHostNames); + this(Map.of(defaultHostResources, hosts.asCollection()), failOnOutOfCapacity, false, false, sharedHosts, defaultHostResources, startIndexForClusters, retiredHostNames); } public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, @@ -107,8 +114,10 @@ public class InMemoryProvisioner implements HostProvisioner { boolean useMaxResources, boolean alwaysReturnOneNode, boolean sharedHosts, + NodeResources defaultResources, int startIndexForClusters, String ... retiredHostNames) { + this.defaultNodeResources = defaultResources; this.failOnOutOfCapacity = failOnOutOfCapacity; this.useMaxResources = useMaxResources; this.alwaysReturnOneNode = alwaysReturnOneNode; @@ -139,9 +148,9 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { - List<Host> defaultHosts = freeNodes.get(defaultResources); + List<Host> defaultHosts = freeNodes.get(defaultHostResources); if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available"); - Host newHost = freeNodes.removeValue(defaultResources, 0); + Host newHost = freeNodes.removeValue(defaultHostResources, 0); return new HostSpec(newHost.hostname(), List.of(alias), Optional.empty()); } @@ -170,7 +179,7 @@ public class InMemoryProvisioner implements HostProvisioner { int nodes = failOnOutOfCapacity || required ? requested.nodes() - : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); + : Math.min(requested.nodes(), freeNodes.get(defaultHostResources).size() + totalAllocatedTo(cluster)); if (alwaysReturnOneNode) nodes = 1; @@ -220,8 +229,15 @@ public class InMemoryProvisioner implements HostProvisioner { host.dockerImageRepo()); } - private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources, + // Minimal capacity policies + private NodeResources decideResources(NodeResources requestedResources) { + if (requestedResources.isUnspecified()) return defaultNodeResources; + return requestedResources; + } + + private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResourcesOrUnspecified, int nodesInGroup, int startIndex, boolean canFail) { + var requestedResources = decideResources(requestedResourcesOrUnspecified); List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); allocations.put(clusterGroup, allocation); diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index ae6f1fd96e4..3d96849fa15 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -24,7 +24,7 @@ public class OnnxModel extends DistributableResource { private String statelessExecutionMode = null; private Integer statelessInterOpThreads = null; private Integer statelessIntraOpThreads = null; - private GpuDevice gpuDevice = null; + private Integer gpuDevice = null; public OnnxModel(String name) { super(name); @@ -114,9 +114,9 @@ public class OnnxModel extends DistributableResource { } } - public void setGpuDevice(int deviceNumber, boolean required) { + public void setGpuDevice(int deviceNumber) { if (deviceNumber >= 0) { - this.gpuDevice = new GpuDevice(deviceNumber, required); + this.gpuDevice = deviceNumber; } } @@ -124,16 +124,8 @@ public class OnnxModel extends DistributableResource { return Optional.ofNullable(statelessIntraOpThreads); } - public Optional<GpuDevice> getGpuDevice() { + public Optional<Integer> getGpuDevice() { return Optional.ofNullable(gpuDevice); } - public record GpuDevice(int deviceNumber, boolean required) { - - public GpuDevice { - if (deviceNumber < 0) throw new IllegalArgumentException("deviceNumber cannot be negative, got " + deviceNumber); - } - - } - } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java index cff0776fddd..f63e872836e 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java @@ -52,10 +52,8 @@ public class FileDistributedOnnxModels { if (model.getStatelessIntraOpThreads().isPresent()) modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); if (model.getGpuDevice().isPresent()) { - modelBuilder.gpu_device(model.getGpuDevice().get().deviceNumber()); - modelBuilder.gpu_device_required(model.getGpuDevice().get().required()); + modelBuilder.gpu_device(model.getGpuDevice().get()); } - builder.model(modelBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index ec941eacce1..ca5931fd9e8 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -70,7 +70,7 @@ public class SummaryClass extends Derived { accessingDiskSummary = true; } addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), - getSource(summaryField), fields); + getSource(summaryField, schema), fields); } } @@ -143,7 +143,7 @@ public class SummaryClass extends Derived { } } - static String getSource(SummaryField summaryField) { + static String getSource(SummaryField summaryField, Schema schema) { if (summaryField.getTransform() == SummaryTransform.NONE) { return ""; } @@ -159,7 +159,7 @@ public class SummaryClass extends Derived { { return summaryField.getSingleSource(); } else if (summaryField.getTransform().isDynamic()) { - return DynamicSummaryTransformUtils.getSource(summaryField); + return DynamicSummaryTransformUtils.getSource(summaryField, schema); } else { return ""; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java index 20597eca64f..50739ad3883 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java @@ -1,6 +1,8 @@ package com.yahoo.schema.processing; import com.yahoo.document.DataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.SummaryField; /** @@ -50,10 +52,22 @@ public class DynamicSummaryTransformUtils { return summaryFieldIsPopulatedBySourceField(summaryField.getDataType()); } - public static String getSource(SummaryField summaryField) { - // Summary fields with the original supported type is always present in the document type, - // and we must use that field as source at run-time. - return isOriginalSupportedType(summaryField.getDataType()) ? - summaryField.getName() : summaryField.getSingleSource(); + public static String getSource(SummaryField summaryField, Schema schema) { + // Summary fields with the original supported type is always present in the document type. + // However, if the source of that summary field is a single explicit source that exists in the schema we + // use that as source instead as this is handled by the backend code. + // This is a move in the right direction to avoid adding some summary fields as extra document fields. + if (isOriginalSupportedType(summaryField.getDataType())) { + if (summaryField.hasExplicitSingleSource()) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null) { + return sourceFieldName; + } + } + return summaryField.getName(); + } else { + return summaryField.getSingleSource(); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 58044163885..7439e65dee6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -276,6 +276,9 @@ public class SummaryField extends Field implements Cloneable, TypedKey { if (sourceName.contains(".")) { return false; } + if (sources.size() > 1) { + return false; + } return true; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 69accef2fe4..d6569dcaa58 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -9,7 +9,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.container.core.documentapi.DocumentAccessProvider; import com.yahoo.container.di.config.PlatformBundlesConfig; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.osgi.provider.model.ComponentModel; @@ -148,7 +147,7 @@ public class ClusterControllerContainer extends Container implements private void configureReindexing() { addFileBundle(REINDEXING_CONTROLLER_BUNDLE.getName()); - addComponent(new SimpleComponent(DocumentAccessProvider.class.getName())); + addComponent(new SimpleComponent("com.yahoo.container.core.documentapi.DocumentAccessProvider")); addComponent("reindexing-maintainer", "ai.vespa.reindexing.ReindexingMaintainer", REINDEXING_CONTROLLER_BUNDLE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java index 9c0e7be452a..63ee5b06dfb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AbstractBundleValidator.java @@ -1,9 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import aQute.bnd.header.Parameters; -import aQute.bnd.osgi.Domain; -import aQute.bnd.version.VersionRange; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; @@ -23,7 +20,6 @@ import java.io.IOException; import java.io.StringReader; import java.nio.file.Paths; import java.util.Optional; -import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.jar.JarFile; import java.util.jar.Manifest; @@ -31,7 +27,7 @@ import java.util.logging.Level; import java.util.regex.Pattern; /** - * Base class for OSGi bundle validator. Uses BND library for some of the validation. + * Base class for OSGi bundle validator. * * @author bjorncs */ @@ -79,14 +75,13 @@ public abstract class AbstractBundleValidator extends Validator { } } - protected final void forEachImportPackage(Manifest mf, BiConsumer<String, VersionRange> consumer) { - Parameters importPackage = Domain.domain(mf).getImportPackage(); - importPackage.forEach((packageName, attrs) -> { - VersionRange versionRange = attrs.getVersion() != null - ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) - : null; - consumer.accept(packageName, versionRange); - }); + protected final void forEachImportPackage(Manifest mf, Consumer<String> consumer) { + String importPackage = mf.getMainAttributes().getValue("Import-Package"); + ImportPackageInfo importPackages = new ImportPackageInfo(importPackage); + + for (String packageName : importPackages.packages()) { + consumer.accept(packageName); + } } protected final void log(DeployState state, Level level, String fmt, Object... args) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index c8a8df18f98..eed88a5fab0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation; -import aQute.bnd.version.VersionRange; import com.yahoo.config.model.deploy.DeployState; import org.w3c.dom.Document; @@ -19,7 +18,6 @@ import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; import java.util.regex.Pattern; -import java.util.stream.Collectors; /** * A validator for bundles. @@ -63,11 +61,10 @@ public class BundleValidator extends AbstractBundleValidator { private void validateImportedPackages(DeployState state, JarFile jar, Manifest manifest) { Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); - forEachImportPackage(manifest, (packageName, versionRange) -> { + forEachImportPackage(manifest, (packageName) -> { for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { - if (matcher.test(packageName) - && (versionRange == null || deprecatedBundle.versionDiscriminator.test(versionRange))) { + if (matcher.test(packageName)) { deprecatedPackagesInUse.computeIfAbsent(deprecatedBundle, __ -> new ArrayList<>()) .add(packageName); } @@ -91,22 +88,15 @@ public class BundleValidator extends AbstractBundleValidator { final String name; final Collection<Predicate<String>> javaPackageMatchers; - final Predicate<VersionRange> versionDiscriminator; final String description; - DeprecatedProvidedBundle(String name, String description, Collection<String> javaPackagePatterns) { - this(name, description, __ -> true, javaPackagePatterns); - } - DeprecatedProvidedBundle(String name, String description, - Predicate<VersionRange> versionDiscriminator, Collection<String> javaPackagePatterns) { this.name = name; this.javaPackageMatchers = javaPackagePatterns.stream() .map(s -> Pattern.compile(s).asMatchPredicate()) .toList(); - this.versionDiscriminator = versionDiscriminator; this.description = description; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 7b380cd5acf..c7a363010b7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.CertificateRemovalChangeValidator; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; import com.yahoo.vespa.model.application.validation.change.CloudAccountChangeValidator; -import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; +import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; import com.yahoo.vespa.model.application.validation.change.ConfigValueChangeValidator; import com.yahoo.vespa.model.application.validation.change.ContainerRestartValidator; import com.yahoo.vespa.model.application.validation.change.ContentClusterRemovalValidator; @@ -23,7 +23,6 @@ import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterC import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator; import com.yahoo.vespa.model.application.validation.change.NodeResourceChangeValidator; import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseValidator; -import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.RedundancyValidator; @@ -116,7 +115,7 @@ public class Validation { new StartupCommandChangeValidator(), new ContentTypeRemovalValidator(), new ContentClusterRemovalValidator(), - new ClusterSizeReductionValidator(), + new ResourcesReductionValidator(), new ResourcesReductionValidator(), new ContainerRestartValidator(), new NodeResourceChangeValidator(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java deleted file mode 100644 index 353be99cfa9..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.change; - -import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.config.application.api.ValidationId; - -import java.util.List; - -/** - * Checks that no cluster sizes are reduced too much in one go. - * - * @author bratseth - */ -public class ClusterSizeReductionValidator implements ChangeValidator { - - @Override - public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { - for (var clusterId : current.allClusters()) { - Capacity currentCapacity = current.provisioned().all().get(clusterId); - Capacity nextCapacity = next.provisioned().all().get(clusterId); - if (currentCapacity == null || nextCapacity == null) continue; - validate(currentCapacity, nextCapacity, clusterId, deployState); - } - return List.of(); - } - - private void validate(Capacity current, Capacity next, ClusterSpec.Id clusterId, DeployState deployState) { - int currentSize = current.minResources().nodes(); - int nextSize = next.minResources().nodes(); - // don't allow more than 50% reduction, but always allow to reduce size with 1 - if ( nextSize < currentSize * 0.5 && nextSize != currentSize - 1) - deployState.validationOverrides().invalid(ValidationId.clusterSizeReduction, - "Size reduction in '" + clusterId.value() + "' is too large: " + - "New min size must be at least 50% of the current min size. " + - "Current size: " + currentSize + ", new size: " + nextSize, - deployState.now()); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index b27266221d2..cbfbf2236d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -4,63 +4,90 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.model.VespaModel; import java.util.List; -import java.util.Locale; -import java.util.Optional; -import java.util.stream.Stream; /** - * Checks that no nodes resources are reduced too much in one go. + * Checks that no cluster sizes are reduced too much in one go. * - * @author freva + * @author bratseth */ public class ResourcesReductionValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { for (var clusterId : current.allClusters()) { - Capacity currentCapacity = current.provisioned().all().get(clusterId); - Capacity nextCapacity = next.provisioned().all().get(clusterId); - if (currentCapacity == null || nextCapacity == null) continue; - validate(currentCapacity, nextCapacity, clusterId, deployState); + if (next.allClusters().contains(clusterId)) + validate(clusterId, current, next, deployState); } - return List.of(); } - private void validate(Capacity current, Capacity next, ClusterSpec.Id clusterId, DeployState deployState) { - if (current.minResources().nodeResources().isUnspecified()) return; - if (next.minResources().nodeResources().isUnspecified()) return; - - List<String> illegalChanges = Stream.of( - validateResource("vCPU", - current.minResources().nodeResources().vcpu(), - next.minResources().nodeResources().vcpu()), - validateResource("memory GB", - current.minResources().nodeResources().memoryGb(), - next.minResources().nodeResources().memoryGb()), - validateResource("disk GB", - current.minResources().nodeResources().diskGb(), - next.minResources().nodeResources().diskGb())) - .flatMap(Optional::stream) - .toList(); - if (illegalChanges.isEmpty()) return; + private void validate(ClusterSpec.Id clusterId, + VespaModel currentModel, + VespaModel nextModel, + DeployState deployState) { + ClusterResources current = clusterResources(clusterId, currentModel); + ClusterResources next = clusterResources(clusterId, nextModel); + if (current == null || next == null) return; // No request recording - test + if (current.nodeResources().isUnspecified() || next.nodeResources().isUnspecified()) { + // Self-hosted - unspecified resources; compare node count + int currentNodes = current.nodes(); + int nextNodes = next.nodes(); + if (nextNodes < 0.5 * currentNodes && nextNodes != currentNodes - 1) { + deployState.validationOverrides().invalid(ValidationId.resourcesReduction, + "Size reduction in '" + clusterId.value() + "' is too large: " + + "To guard against mistakes, the new max nodes must be at least 50% of the current nodes. " + + "Current nodes: " + currentNodes + ", new nodes: " + nextNodes, + deployState.now()); + } + } + else { + NodeResources currentResources = current.totalResources(); + NodeResources nextResources = next.totalResources(); + if (nextResources.vcpu() < 0.5 * currentResources.vcpu() || + nextResources.memoryGb() < 0.5 * currentResources.memoryGb() || + nextResources.diskGb() < 0.5 * currentResources.diskGb()) + deployState.validationOverrides().invalid(ValidationId.resourcesReduction, + "Resource reduction in '" + clusterId.value() + "' is too large: " + + "To guard against mistakes, the new max resources must be at least 50% of the current " + + "max resources in all dimensions. " + + "Current: " + currentResources.withBandwidthGbps(0) + // (don't output bandwidth here) + ", new: " + nextResources.withBandwidthGbps(0), + deployState.now()); + } - deployState.validationOverrides().invalid(ValidationId.resourcesReduction, - "Resource reduction in '" + clusterId.value() + "' is too large. " + - String.join(" ", illegalChanges) + - " New min resources must be at least 50% of the current min resources", - deployState.now()); } - private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) { - // don't allow more than 50% reduction, but always allow to reduce by 1 - if (nextValue >= currentValue * 0.5 || nextValue >= currentValue - 1) return Optional.empty(); - return Optional.of(String.format(Locale.ENGLISH ,"Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue)); + /** + * If the given requested cluster resources does not specify node resources, return them with + * the current node resources of the cluster, as that is what unspecified resources actually resolved to. + * This will always yield specified node resources on hosted instances and never on self-hosted instances. + */ + private ClusterResources clusterResources(ClusterSpec.Id id, VespaModel model) { + if ( ! model.provisioned().all().containsKey(id)) return null; + + ClusterResources resources = model.provisioned().all().get(id).maxResources(); + if ( ! resources.nodeResources().isUnspecified()) return resources; + + var containerCluster = model.getContainerClusters().get(id.value()); + if (containerCluster != null) { + if ( ! containerCluster.getContainers().isEmpty()) + return resources.with(containerCluster.getContainers().get(0).getHostResource().advertisedResources()); + } + + var contentCluster = model.getContentClusters().get(id.value()); + if (contentCluster != null) { + var searchCluster = contentCluster.getSearch(); + if ( ! searchCluster.getSearchNodes().isEmpty()) + return resources.with(searchCluster.getSearchNodes().get(0).getHostResource().advertisedResources()); + } + + return resources; // only expected for admin clusters } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c0182c2f5ac..2f41c177ea5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -112,7 +112,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); - addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); + addSimpleComponent("com.yahoo.container.core.documentapi.DocumentAccessProvider"); addSimpleComponent(DOCUMENT_TYPE_MANAGER_CLASS); addMetricsHandlers(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index e07b1a95e20..700393e84f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -685,12 +685,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { onnxModel.setStatelessExecutionMode(getStringValue(modelElement, "execution-mode", null)); onnxModel.setStatelessInterOpThreads(getIntValue(modelElement, "interop-threads", -1)); onnxModel.setStatelessIntraOpThreads(getIntValue(modelElement, "intraop-threads", -1)); - Element gpuDeviceElement = XML.getChild(modelElement, "gpu-device"); - if (gpuDeviceElement != null) { - int gpuDevice = Integer.parseInt(gpuDeviceElement.getTextContent()); - boolean required = Boolean.parseBoolean(extractAttribute(gpuDeviceElement, "required")); - onnxModel.setGpuDevice(gpuDevice, required); - } + onnxModel.setGpuDevice(getIntValue(modelElement, "gpu-device", -1)); } cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 2053ac93aba..2d8d89e3bd2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; @@ -21,6 +20,7 @@ import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; @@ -48,6 +48,7 @@ import com.yahoo.vespa.model.content.StorageGroup; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; +import com.yahoo.vespa.model.routing.DocumentProtocol; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -155,10 +156,6 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce if (tuning != null) setupTuning(c, tuning); - ModelElement experimental = contentElement.child("experimental"); - if (experimental != null) - setupExperimental(c, experimental); - if (context.getParentProducer().getRoot() == null) return c; addClusterControllers(context, contentElement, c, deployState); @@ -249,10 +246,6 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce return 0.0; } - private void setupExperimental(ContentCluster cluster, ModelElement experimental) { - // Put handling of experimental flags here - } - private void validateGroupSiblings(String cluster, StorageGroup group) { Set<String> siblings = new HashSet<>(); for (StorageGroup g : group.getSubgroups()) { @@ -302,7 +295,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } else { // self-hosted: Put cluster controller on config servers or use explicit cluster controllers if (admin.getClusterControllers() == null) { - var hosts = admin.getConfigservers().stream().map(s -> s.getHostResource()).toList(); + var hosts = admin.getConfigservers().stream().map(AbstractService::getHostResource).toList(); if (hosts.size() > 1) { var message = "When having content clusters and more than 1 config server " + "it is recommended to configure cluster controllers explicitly."; @@ -447,16 +440,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce @Override public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { if ( ! getSearch().hasIndexedCluster()) return; - builder.defaultroute(com.yahoo.vespa.model.routing.DocumentProtocol.getDirectRouteName(getConfigId())) - .route(new MessagetyperouteselectorpolicyConfig.Route.Builder() - .messagetype(DocumentProtocol.MESSAGE_PUTDOCUMENT) - .name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))) - .route(new MessagetyperouteselectorpolicyConfig.Route.Builder() - .messagetype(DocumentProtocol.MESSAGE_REMOVEDOCUMENT) - .name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))) - .route(new MessagetyperouteselectorpolicyConfig.Route.Builder() - .messagetype(DocumentProtocol.MESSAGE_UPDATEDOCUMENT) - .name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))); + DocumentProtocol.getConfig(builder, getConfigId()); } public com.yahoo.vespa.model.content.StorageGroup getRootGroup() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index 8a8d38e23e3..ad0312705ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -9,6 +9,7 @@ import com.yahoo.messagebus.routing.ApplicationSpec; import com.yahoo.messagebus.routing.HopSpec; import com.yahoo.messagebus.routing.RouteSpec; import com.yahoo.messagebus.routing.RoutingTableSpec; +import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; @@ -118,6 +119,19 @@ public final class DocumentProtocol implements Protocol, } } + public static void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder, String configId) { + builder.defaultroute(getDirectRouteName(configId)) + .route(new MessagetyperouteselectorpolicyConfig.Route.Builder() + .messagetype(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.MESSAGE_PUTDOCUMENT) + .name(getIndexedRouteName(configId))) + .route(new MessagetyperouteselectorpolicyConfig.Route.Builder() + .messagetype(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.MESSAGE_REMOVEDOCUMENT) + .name(getIndexedRouteName(configId))) + .route(new MessagetyperouteselectorpolicyConfig.Route.Builder() + .messagetype(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.MESSAGE_UPDATEDOCUMENT) + .name(getIndexedRouteName(configId))); + } + private static void addRoutes(String directRoute, String indexedRoute, DocumentProtocolPoliciesConfig.Cluster.Builder builder) { builder.defaultRoute(directRoute) .route(new DocumentProtocolPoliciesConfig.Cluster.Route.Builder() diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 888851b2db2..b8c02b013aa 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -105,7 +105,8 @@ ModelEvaluation = element model-evaluation { element interop-threads { xsd:nonNegativeInteger }? & element execution-mode { string "sequential" | string "parallel" }? & element gpu-device { - attribute required { xsd:boolean } & + # TODO(mpolden): Remove this after 2023-02-01 + attribute required { xsd:boolean }? & xsd:nonNegativeInteger }? }* diff --git a/config-model/src/test/cfg/application/onnx/services.xml b/config-model/src/test/cfg/application/onnx/services.xml index 68c2e8530be..088bbcc4921 100644 --- a/config-model/src/test/cfg/application/onnx/services.xml +++ b/config-model/src/test/cfg/application/onnx/services.xml @@ -8,11 +8,11 @@ <models> <model name="mul"> <intraop-threads>2</intraop-threads> - <gpu-device required="false">0</gpu-device> </model> <model name="non-existent-model"> <interop-threads>400</interop-threads> <execution-mode>parallel</execution-mode> + <gpu-device>0</gpu-device> </model> </models> </onnx> diff --git a/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg b/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg index 3703bb8b218..c5d5b86de67 100644 --- a/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg +++ b/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg @@ -35,13 +35,13 @@ classes[].fields[].command "summaryfeatures" classes[].fields[].source "" classes[].fields[].name "str_3_dyn" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "str_3_dyn" +classes[].fields[].source "str_3" classes[].fields[].name "arr_3_dyn" classes[].fields[].command "dynamicteaser" classes[].fields[].source "arr_3" classes[].fields[].name "str_4_bold" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "str_4_bold" +classes[].fields[].source "str_4" classes[].fields[].name "arr_4_bold" classes[].fields[].command "dynamicteaser" classes[].fields[].source "arr_4" @@ -53,13 +53,13 @@ classes[].name "dyn" classes[].omitsummaryfeatures false classes[].fields[].name "str_3_dyn" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "str_3_dyn" +classes[].fields[].source "str_3" classes[].fields[].name "arr_3_dyn" classes[].fields[].command "dynamicteaser" classes[].fields[].source "arr_3" classes[].fields[].name "str_4_bold" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "str_4_bold" +classes[].fields[].source "str_4" classes[].fields[].name "arr_4_bold" classes[].fields[].command "dynamicteaser" classes[].fields[].source "arr_4" diff --git a/config-model/src/test/derived/multiplesummaries/summary.cfg b/config-model/src/test/derived/multiplesummaries/summary.cfg index 72543b46c8e..0b2be2b3ab1 100644 --- a/config-model/src/test/derived/multiplesummaries/summary.cfg +++ b/config-model/src/test/derived/multiplesummaries/summary.cfg @@ -26,7 +26,7 @@ classes[].fields[].command "dynamicteaser" classes[].fields[].source "d" classes[].fields[].name "dynamice" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "dynamice" +classes[].fields[].source "e" classes[].fields[].name "f" classes[].fields[].command "" classes[].fields[].source "" @@ -47,7 +47,7 @@ classes[].fields[].command "" classes[].fields[].source "" classes[].fields[].name "adynamic2" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "adynamic2" +classes[].fields[].source "a" classes[].fields[].name "alltags" classes[].fields[].command "" classes[].fields[].source "" @@ -59,10 +59,10 @@ classes[].fields[].command "" classes[].fields[].source "" classes[].fields[].name "abolded2" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "abolded2" +classes[].fields[].source "a" classes[].fields[].name "aboldeddynamic" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "aboldeddynamic" +classes[].fields[].source "a" classes[].fields[].name "documentid" classes[].fields[].command "documentid" classes[].fields[].source "" @@ -131,7 +131,7 @@ classes[].name "anothernotattributesonly2" classes[].omitsummaryfeatures false classes[].fields[].name "adynamic2" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "adynamic2" +classes[].fields[].source "a" classes[].fields[].name "c" classes[].fields[].command "attribute" classes[].fields[].source "c" @@ -209,7 +209,7 @@ classes[].name "notattributesonly4" classes[].omitsummaryfeatures false classes[].fields[].name "abolded2" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "abolded2" +classes[].fields[].source "a" classes[].fields[].name "c" classes[].fields[].command "attribute" classes[].fields[].source "c" @@ -224,7 +224,7 @@ classes[].name "notattributesonly5" classes[].omitsummaryfeatures false classes[].fields[].name "aboldeddynamic" classes[].fields[].command "dynamicteaser" -classes[].fields[].source "aboldeddynamic" +classes[].fields[].source "a" classes[].fields[].name "c" classes[].fields[].command "attribute" classes[].fields[].source "c" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index 2071d142da9..b5ebefdbd9c 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -77,11 +77,11 @@ public class SummaryTestCase extends AbstractSchemaTestCase { assertSummaryField("exact", SummaryClassField.Type.LONGSTRING, fields.next()); assertSummaryField("title", SummaryClassField.Type.LONGSTRING, fields.next()); assertSummaryField("description", SummaryClassField.Type.LONGSTRING, fields.next()); - assertSummaryField("dyndesc", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "dyndesc", fields.next()); + assertSummaryField("dyndesc", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "description", fields.next()); assertSummaryField("longdesc", SummaryClassField.Type.LONGSTRING, fields.next()); assertSummaryField("longstat", SummaryClassField.Type.LONGSTRING, fields.next()); - assertSummaryField("dynlong", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "dynlong", fields.next()); - assertSummaryField("dyndesc2", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "dyndesc2", fields.next()); + assertSummaryField("dynlong", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "longdesc", fields.next()); + assertSummaryField("dyndesc2", SummaryClassField.Type.LONGSTRING, "dynamicteaser", "longdesc", fields.next()); assertSummaryField("measurement", SummaryClassField.Type.INTEGER, "attribute", "measurement", fields.next()); assertSummaryField("rankfeatures", SummaryClassField.Type.FEATUREDATA, "rankfeatures", fields.next()); assertSummaryField("summaryfeatures", SummaryClassField.Type.FEATUREDATA, "summaryfeatures", fields.next()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index 2d4211c693b..5a0f0a589fd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -112,6 +112,8 @@ public class BundleValidatorTest { "java.nio;version=\"[0.0.0,1)\",java.util.concurrent;version=\"[0.0.0,1)\",java.util;version=\"[0.0.0,1)\"").packages(), List.of("com.yahoo.config", "com.yahoo.filedistribution.fileacquirer", "com.yahoo.jdisc", "com.yahoo.jdisc.handler", "java.io", "java.lang", "java.nio", "java.util.concurrent", "java.util")); + assertEquals(new ImportPackageInfo("org.json;version=\"[0.0.0,1)\",org.eclipse.jetty.client.api;version=\"[9.4.46,10)").packages(), + List.of("org.json", "org.eclipse.jetty.client.api")); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java deleted file mode 100644 index 7e172171052..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation.change; - -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ValidationTester; -import com.yahoo.yolean.Exceptions; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - -/** - * @author bratseth - */ -public class ClusterSizeReductionValidatorTest { - - @Test - void testSizeReductionValidation() { - ValidationTester tester = new ValidationTester(33); - - VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst(); - try { - tester.deploy(previous, getServices(14), Environment.prod, null); - fail("Expected exception due to cluster size reduction"); - } - catch (IllegalArgumentException expected) { - assertEquals("cluster-size-reduction: Size reduction in 'default' is too large: " + - "New min size must be at least 50% of the current min size. " + - "Current size: 30, new size: 14. " + - ValidationOverrides.toAllowMessage(ValidationId.clusterSizeReduction), - Exceptions.toMessageString(expected)); - } - } - - @Test - void testSizeReductionValidationMinimalDecreaseIsAllowed() { - ValidationTester tester = new ValidationTester(30); - - VespaModel previous = tester.deploy(null, getServices(3), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(2), Environment.prod, null); - } - - @Test - void testOverridingSizereductionValidation() { - ValidationTester tester = new ValidationTester(33); - - VespaModel previous = tester.deploy(null, getServices(30), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(14), Environment.prod, sizeReductionOverride); // Allowed due to override - } - - private static String getServices(int size) { - return "<services version='1.0'>" + - " <content id='default' version='1.0'>" + - " <redundancy>1</redundancy>" + - " <engine>" + - " <proton/>" + - " </engine>" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <nodes count='" + size + "'/>" + - " </content>" + - "</services>"; - } - - private static final String sizeReductionOverride = - "<validation-overrides>\n" + - " <allow until='2000-01-03'>cluster-size-reduction</allow>\n" + - "</validation-overrides>\n"; - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index 651f87c614b..4c185e9dfb8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -16,86 +16,221 @@ import static org.junit.jupiter.api.Assertions.fail; /** * @author freva + * @author bratseth */ public class ResourcesReductionValidatorTest { - private final InMemoryProvisioner provisioner = new InMemoryProvisioner(30, new NodeResources(64, 128, 1000, 10), false); + private final NodeResources hostResources = new NodeResources(64, 128, 1000, 10); + private final InMemoryProvisioner provisioner = new InMemoryProvisioner(30, hostResources, true, InMemoryProvisioner.defaultHostResources); + private final InMemoryProvisioner provisionerSelfHosted = new InMemoryProvisioner(30, hostResources, true, NodeResources.unspecified()); + private final NodeResources defaultResources = InMemoryProvisioner.defaultHostResources; private final ValidationTester tester = new ValidationTester(provisioner); @Test void fail_when_reduction_by_over_50_percent() { - VespaModel previous = tester.deploy(null, getServices(new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst(); + var fromResources = new NodeResources(8, 64, 800, 1); + var toResources = new NodeResources(8, 16, 800, 1); + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null).getFirst(); try { - tester.deploy(previous, getServices(new NodeResources(8, 16, 800, 1)), Environment.prod, null); + tester.deploy(previous, contentServices(6, toResources), Environment.prod, null); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { - assertEquals("resources-reduction: Resource reduction in 'default' is too large. " + - "Current memory GB: 64.00, new: 16.00. New min resources must be at least 50% of the current min resources. " + - ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction), - Exceptions.toMessageString(expected)); + assertResourceReductionException(expected, + fromResources.multipliedBy(6), + toResources.multipliedBy(6)); } } @Test void fail_when_reducing_multiple_resources_by_over_50_percent() { - VespaModel previous = tester.deploy(null, getServices(new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst(); + var fromResources = new NodeResources(8, 64, 800, 1); + var toResources = new NodeResources(3, 16, 200, 1); + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null).getFirst(); try { - tester.deploy(previous, getServices(new NodeResources(3, 16, 200, 1)), Environment.prod, null); + tester.deploy(previous, contentServices(6, toResources), Environment.prod, null); fail("Expected exception due to resources reduction"); } catch (IllegalArgumentException expected) { - assertEquals("resources-reduction: Resource reduction in 'default' is too large. " + - "Current vCPU: 8.00, new: 3.00. Current memory GB: 64.00, new: 16.00. Current disk GB: 800.00, new: 200.00. " + - "New min resources must be at least 50% of the current min resources. " + - ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction), - Exceptions.toMessageString(expected)); + assertResourceReductionException(expected, + fromResources.multipliedBy(6), + toResources.multipliedBy(6)); } } @Test void small_resource_decrease_is_allowed() { - VespaModel previous = tester.deploy(null, getServices(new NodeResources(1.5, 64, 800, 1)), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(new NodeResources(.5, 48, 600, 1)), Environment.prod, null); + VespaModel previous = tester.deploy(null, contentServices(6, new NodeResources(1.5, 64, 800, 1)), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(6, new NodeResources(.5, 48, 600, 1)), Environment.prod, null); + } + + @Test + void reorganizing_resources_is_allowed() { + VespaModel previous = tester.deploy(null, contentServices(12, new NodeResources(2, 10, 100, 1)), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(4, new NodeResources(6, 30, 300, 1)), Environment.prod, null); } @Test void overriding_resource_decrease() { - VespaModel previous = tester.deploy(null, getServices(new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(new NodeResources(8, 16, 800, 1)), Environment.prod, resourcesReductionOverride); // Allowed due to override + VespaModel previous = tester.deploy(null, contentServices(6, new NodeResources(8, 64, 800, 1)), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(6, new NodeResources(8, 16, 800, 1)), Environment.prod, resourcesReductionOverride); // Allowed due to override } @Test - void allowed_to_go_to_not_specifying_resources() { - VespaModel previous = tester.deploy(null, getServices(new NodeResources(1.5, 64, 800, 1)), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(null), Environment.prod, null); + void reduction_is_detected_when_going_from_unspecified_resources_container() { + NodeResources toResources = defaultResources.withDiskGb(defaultResources.diskGb() / 5); + try { + VespaModel previous = tester.deploy(null, containerServices(6, null), Environment.prod, null).getFirst(); + tester.deploy(previous, containerServices(6, toResources), Environment.prod, null); + fail("Expected exception due to resources reduction"); + } + catch (IllegalArgumentException expected) { + assertResourceReductionException(expected, + defaultResources.multipliedBy(6), + toResources.multipliedBy(6)); + } } @Test - void allowed_to_go_from_not_specifying_resources() { - VespaModel previous = tester.deploy(null, getServices(null), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices(new NodeResources(1.5, 64, 800, 1)), Environment.prod, null); + void reduction_is_detected_when_going_to_unspecified_resources_container() { + NodeResources fromResources = defaultResources.withVcpu(defaultResources.vcpu() * 3); + try { + VespaModel previous = tester.deploy(null, containerServices(6, fromResources), Environment.prod, null).getFirst(); + tester.deploy(previous, containerServices(6, null), Environment.prod, null); + fail("Expected exception due to resources reduction"); + } + catch (IllegalArgumentException expected) { + assertResourceReductionException(expected, + fromResources.multipliedBy(6), + defaultResources.multipliedBy(6)); + } } - private static String getServices(NodeResources resources) { + @Test + void reduction_is_detected_when_going_from_unspecified_resources_content() { + NodeResources toResources = defaultResources.withDiskGb(defaultResources.diskGb() / 5); + try { + VespaModel previous = tester.deploy(null, contentServices(6, null), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(6, toResources), Environment.prod, null); + fail("Expected exception due to resources reduction"); + } + catch (IllegalArgumentException expected) { + assertResourceReductionException(expected, + defaultResources.multipliedBy(6), + toResources.multipliedBy(6)); + } + } + + @Test + void reduction_is_detected_when_going_to_unspecified_resources_content() { + NodeResources fromResources = defaultResources.withVcpu(defaultResources.vcpu() * 3); + try { + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(6, null), Environment.prod, null); + fail("Expected exception due to resources reduction"); + } + catch (IllegalArgumentException expected) { + assertResourceReductionException(expected, + fromResources.multipliedBy(6), + defaultResources.multipliedBy(6)); + } + } + + @Test + void testSizeReductionValidationWithUnspecifiedResourcesHosted() { + int fromNodes = 30; + int toNodes = 14; + try { + ValidationTester tester = new ValidationTester(33); + VespaModel previous = tester.deploy(null, contentServices(fromNodes, null), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(toNodes, null), Environment.prod, null); + fail("Expected exception due to resources reduction"); + } + catch (IllegalArgumentException expected) { + assertResourceReductionException(expected, + defaultResources.multipliedBy(fromNodes), + defaultResources.multipliedBy(toNodes)); + } + } + + /** Emulate a self-hosted setup in only the sense that it does not set node resources on the provisioned hosts. */ + @Test + void testSizeReductionValidationSelfhosted() { + var tester = new ValidationTester(provisionerSelfHosted); + + VespaModel previous = tester.deploy(null, contentServices(10, null), Environment.prod, null).getFirst(); + try { + tester.deploy(previous, contentServices(4, null), Environment.prod, null); + fail("Expected exception due to resources reduction"); + } + catch (IllegalArgumentException expected) { + assertEquals("resources-reduction: Size reduction in 'default' is too large: " + + "To guard against mistakes, the new max nodes must be at least 50% of the current nodes. " + + "Current nodes: 10, new nodes: 4. " + + ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction), + Exceptions.toMessageString(expected)); + } + } + + @Test + void testSizeReductionValidationMinimalDecreaseIsAllowed() { + ValidationTester tester = new ValidationTester(30); + + VespaModel previous = tester.deploy(null, contentServices(3, null), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(2, null), Environment.prod, null); + } + + @Test + void testOverridingSizeReductionValidation() { + ValidationTester tester = new ValidationTester(33); + + VespaModel previous = tester.deploy(null, contentServices(30, null), Environment.prod, null).getFirst(); + tester.deploy(previous, contentServices(14, null), Environment.prod, resourcesReductionOverride); // Allowed due to override + } + + private void assertResourceReductionException(Exception e, NodeResources currentResources, NodeResources newResources) { + assertEquals("resources-reduction: Resource reduction in 'default' is too large: " + + "To guard against mistakes, the new max resources must be at least 50% of the current max " + + "resources in all dimensions. " + + "Current: " + currentResources.withBandwidthGbps(0) + + ", new: " + newResources.withBandwidthGbps(0) + ". " + + ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction), + Exceptions.toMessageString(e)); + } + + private static String containerServices(int nodes, NodeResources resources) { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", resources.vcpu(), resources.memoryGb(), resources.diskGb()); return "<services version='1.0'>" + - " <content id='default' version='1.0'>" + - " <redundancy>1</redundancy>" + - " <engine>" + - " <proton/>" + - " </engine>" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <nodes count='5'>" + + " <container id='default' version='1.0'>" + + " <nodes count='" + nodes + "'>" + resourcesStr + " </nodes>" + - " </content>" + + " </container>" + "</services>"; } + private static String contentServices(int nodes, NodeResources resources) { + String resourcesStr = resources == null ? + "" : + String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", + resources.vcpu(), resources.memoryGb(), resources.diskGb()); + return "<services version='1.0'>" + + " <content id='default' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <engine>" + + " <proton/>" + + " </engine>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes count='" + nodes + "'>" + + resourcesStr + + " </nodes>" + + " </content>" + + "</services>"; + } + private static final String resourcesReductionOverride = "<validation-overrides>\n" + " <allow until='2000-01-03'>resources-reduction</allow>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index 8ccbe99f70a..b1e28649e9f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -129,8 +129,7 @@ public class StatelessOnnxEvaluationTest { assertEquals(2, mulModel.stateless_intraop_threads()); assertEquals(-1, mulModel.stateless_interop_threads()); assertEquals("", mulModel.stateless_execution_mode()); - assertFalse(mulModel.gpu_device_required()); - assertEquals(0, mulModel.gpu_device()); + assertEquals(-1, mulModel.gpu_device()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index a31d4cd4e20..48ddf6b8a82 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -71,7 +71,7 @@ public class VespaModelTester { } /** Adds some nodes with resources 1, 3, 10 */ - public Hosts addHosts(int count) { return addHosts(InMemoryProvisioner.defaultResources, count); } + public Hosts addHosts(int count) { return addHosts(InMemoryProvisioner.defaultHostResources, count); } public Hosts addHosts(NodeResources resources, int count) { return addHosts(Optional.of(new Flavor(resources)), resources, count); @@ -197,6 +197,7 @@ public class VespaModelTester { useMaxResources, alwaysReturnOneNode, false, + NodeResources.unspecified(), startIndexForClusters, retiredHostNames); provisioner.setEnvironment(zone.environment()); diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index b447cb792a6..ab9418ec488 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -16,6 +16,13 @@ <dependencies> <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 2f2310c3703..8b2bf9fcbcc 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -311,6 +311,13 @@ public class NodeResources { this.gpuResources.plus(other.gpuResources)); } + public NodeResources multipliedBy(double factor) { + return this.withVcpu(vcpu * factor) + .withMemoryGb(memoryGb * factor) + .withDiskGb(diskGb * factor) + .withBandwidthGbps(bandwidthGbps * factor); + } + private boolean isInterchangeableWith(NodeResources other) { ensureSpecified(); other.ensureSpecified(); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java index 0399664faf2..bb864fa1708 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigVerification.java @@ -10,12 +10,13 @@ import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Stack; /** * Tool to verify that configs across multiple config servers are the same. @@ -42,17 +43,16 @@ public class ConfigVerification { } } - private static Map<String, Stack<String>> listConfigs(List<String> urls, CloseableHttpClient httpClient) throws IOException { + private static Map<String, Deque<String>> listConfigs(List<String> urls, CloseableHttpClient httpClient) throws IOException { Map<String, String> outputs = performRequests(urls, httpClient); - Map<String, Stack<String>> recurseMappings = new LinkedHashMap<>(); + Map<String, Deque<String>> recurseMappings = new LinkedHashMap<>(); for (Map.Entry<String, String> entry : outputs.entrySet()) { Slime slime = SlimeUtils.jsonToSlime(entry.getValue()); final List<String> list = new ArrayList<>(); slime.get().field("configs").traverse((ArrayTraverser) (idx, inspector) -> list.add(inspector.asString())); - Stack<String> stack = new Stack<>(); Collections.sort(list); - stack.addAll(list); + Deque<String> stack = new ArrayDeque<>(list); recurseMappings.put(entry.getKey(), stack); } return recurseMappings; @@ -66,10 +66,10 @@ public class ConfigVerification { return outputs; } - private static int compareConfigs(Map<String, Stack<String>> mappings, CloseableHttpClient httpClient) throws IOException { + private static int compareConfigs(Map<String, Deque<String>> mappings, CloseableHttpClient httpClient) throws IOException { for (int n = 0; n < mappings.values().iterator().next().size(); n++) { List<String> recurseUrls = new ArrayList<>(); - for (Map.Entry<String, Stack<String>> entry : mappings.entrySet()) { + for (Map.Entry<String, Deque<String>> entry : mappings.entrySet()) { recurseUrls.add(entry.getValue().pop()); } if ( ! equalOutputs(performRequests(recurseUrls, httpClient))) diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 8f7aae17e84..3107f9029d8 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -11,6 +11,9 @@ import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.zstd; /** * Keeps track of file distribution and url download rpc servers. @@ -43,7 +46,7 @@ public class FileDistributionAndUrlDownload { } private Set<CompressionType> acceptedCompressionTypes() { - Set<CompressionType> acceptedCompressionTypes = Set.of(CompressionType.gzip); + Set<CompressionType> acceptedCompressionTypes = Set.of(gzip, lz4, zstd); String env = System.getenv("VESPA_FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES"); if (env != null && ! env.isEmpty()) { String[] types = env.split(","); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java deleted file mode 100644 index 16a3ef3371d..00000000000 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import com.yahoo.config.codegen.CNode; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.config.codegen.LeafCNode; -import com.yahoo.slime.*; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.util.ConfigUtils; - -import java.io.*; -import java.util.Stack; - -/** - * @author Ulf Lilleengen - */ -public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { - - private final InnerCNode root; - private DataOutputStream out = null; - private Stack<Node> nodeStack; - - public ConfigFileFormat(InnerCNode root) { - this.root = root; - this.nodeStack = new Stack<>(); - } - - private void printPrefix() throws IOException { - for (Node node : nodeStack) { - CNode cnode = node.node; - if (cnode != root) { - encodeString(cnode.getName()); - if (cnode.isArray) { - encodeString("[" + node.arrayIndex + "]"); - if (!(cnode instanceof LeafCNode)) { - encodeString("."); - } - } else if (cnode.isMap) { - encodeString("{\"" + node.mapKey + "\"}"); - if (!(cnode instanceof LeafCNode)) { - encodeString("."); - } - } else if (cnode instanceof LeafCNode) { - encodeString(""); - } else { - encodeString("."); - } - } - } - encodeString(" "); - } - - private void encode(Inspector inspector, CNode node) throws IOException { - switch (inspector.type()) { - case BOOL: - encodeValue(String.valueOf(inspector.asBool()), (LeafCNode) node); - return; - case LONG: - encodeValue(String.valueOf(inspector.asLong()), (LeafCNode) node); - return; - case DOUBLE: - encodeValue(String.valueOf(inspector.asDouble()), (LeafCNode) node); - return; - case STRING: - encodeValue(inspector.asString(), (LeafCNode) node); - return; - case ARRAY: - encodeArray(inspector, node); - return; - case OBJECT: - if (node.isMap) { - encodeMap(inspector, node); - } else { - encodeObject(inspector, node); - } - return; - case NIX: - case DATA: - throw new IllegalArgumentException("Illegal config format supplied. Unknown type for field '" + node.getName() + "'"); - } - throw new RuntimeException("Should not be reached"); - } - - private void encodeMap(Inspector inspector, final CNode node) { - inspector.traverse(new ObjectTraverser() { - @Override - public void field(String name, Inspector inspector) { - try { - nodeStack.push(new Node(node, -1, name)); - if (inspector.type().equals(Type.OBJECT)) { - encodeObject(inspector, node); - } else { - encode(inspector, node); - } - nodeStack.pop(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - } - - private void encodeArray(Inspector inspector, final CNode node) { - inspector.traverse(new ArrayTraverser() { - @Override - public void entry(int idx, Inspector inspector) { - try { - nodeStack.push(new Node(node, idx, "")); - encode(inspector, node); - nodeStack.pop(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - - } - - private void encodeObject(Inspector inspector, CNode node) { - if (!node.isArray && !node.isMap) { - nodeStack.push(new Node(node)); - inspector.traverse(this); - nodeStack.pop(); - } else { - inspector.traverse(this); - } - } - - private void encodeValue(String value, LeafCNode node) throws IOException { - printPrefix(); - try { - if (node instanceof LeafCNode.StringLeaf) { - encodeStringQuoted(value); - } else if (node instanceof LeafCNode.IntegerLeaf) { - //Integer.parseInt(value); - encodeString(value); - } else if (node instanceof LeafCNode.LongLeaf) { - //Long.parseLong(value); - encodeString(value); - } else if (node instanceof LeafCNode.DoubleLeaf) { - //Double.parseDouble(value); - encodeString(value); - } else if (node instanceof LeafCNode.BooleanLeaf) { - encodeString(String.valueOf(Boolean.parseBoolean(value))); - } else if (node instanceof LeafCNode.EnumLeaf) { - // LeafCNode.EnumLeaf enumNode = (LeafCNode.EnumLeaf) node; - // TODO: Reenable this when we can return illegal config id. - // checkLegalEnumValue(enumNode, value); - encodeString(value); - } else { - encodeStringQuoted(value); - } - encodeString("\n"); - } catch (Exception e) { - throw new IllegalArgumentException("Unable to serialize field '" + node.getFullName() + "': ", e); - } - } - - private void checkLegalEnumValue(LeafCNode.EnumLeaf enumNode, String value) { - boolean found = false; - for (String legalVal : enumNode.getLegalValues()) { - if (legalVal.equals(value)) { - found = true; - } - } - if (!found) - throw new IllegalArgumentException("Illegal enum value '" + value + "'"); - } - - private void encodeStringQuoted(String s) throws IOException { - encodeString("\"" + escapeString(s) + "\""); - } - - private String escapeString(String s) { - return ConfigUtils.escapeConfigFormatValue(s); - } - - private void encodeString(String s) throws IOException { - out.write(Utf8.toBytes(s)); - } - - @Override - public void encode(OutputStream os, Slime slime) throws IOException { - encode(os, slime.get()); - } - - private void encode(OutputStream os, Inspector inspector) throws IOException { - this.out = new DataOutputStream(os); - this.nodeStack = new Stack<>(); - nodeStack.push(new Node(root)); - encode(inspector, root); - } - - @Override - public void field(String fieldName, Inspector inspector) { - try { - Node parent = nodeStack.peek(); - CNode child = parent.node.getChild(fieldName); - if (child == null) { - return; // Skip this field to optimistic - } - if (!child.isArray && !child.isMap && child instanceof LeafCNode) { - nodeStack.push(new Node(child)); - encode(inspector, child); - nodeStack.pop(); - } else { - encode(inspector, child); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private class Node { - final int arrayIndex; - final String mapKey; - final CNode node; - Node(CNode node, int arrayIndex, String mapKey) { - this.node = node; - this.arrayIndex = arrayIndex; - this.mapKey = mapKey; - } - - Node(CNode node) { - this(node, -1, ""); - } - } -} diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 3705c167960..e86b13b2c98 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -8,7 +8,6 @@ import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; import java.io.File; import java.lang.reflect.Constructor; @@ -17,11 +16,12 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.Stack; import java.util.logging.Logger; import static java.util.logging.Level.INFO; @@ -42,7 +42,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private final ConfigInstance.Builder rootBuilder; private final ConfigTransformer.PathAcquirer pathAcquirer; private final UrlDownloader urlDownloader; - private final Stack<NamedBuilder> stack = new Stack<>(); + private final Deque<NamedBuilder> stack = new ArrayDeque<>(); public ConfigPayloadApplier(T builder) { this(builder, new IdentityPathAcquirer(), null); @@ -67,27 +67,17 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private void handleValue(Inspector inspector) { switch (inspector.type()) { - case NIX: - case BOOL: - case LONG: - case DOUBLE: - case STRING: - case DATA: - handleLeafValue(inspector); - break; - case ARRAY: - handleARRAY(inspector); - break; - case OBJECT: - handleOBJECT(inspector); - break; - default: + case NIX, BOOL, LONG, DOUBLE, STRING, DATA -> handleLeafValue(inspector); + case ARRAY -> handleARRAY(inspector); + case OBJECT -> handleOBJECT(inspector); + default -> { assert false : "Should not be reached"; + } } } private void handleARRAY(Inspector inspector) { - inspector.traverse((ArrayTraverser)(int index, Inspector value) -> handleArrayEntry(index, value)); + inspector.traverse((ArrayTraverser) this::handleArrayEntry); } private void handleArrayEntry(int idx, Inspector inspector) { @@ -108,11 +98,11 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } private void handleOBJECT(Inspector inspector) { - inspector.traverse((String name, Inspector value) -> handleObjectEntry(name, value)); + inspector.traverse(this::handleObjectEntry); NamedBuilder builder = stack.pop(); // Need to set e.g struct(Struct.Builder) here - if ( ! stack.empty()) { + if ( ! stack.isEmpty()) { try { invokeSetter(stack.peek().builder, builder.peekName(), builder.builder); } catch (Exception e) { @@ -165,7 +155,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { throw new RuntimeException("Missing map builder (this should never happen): " + stack.peek()); setMapLeafValue(name, builder.builder()); stack.push(builder); - inspector.traverse((ObjectTraverser) (key, value) -> handleObjectEntry(key, value)); + inspector.traverse(this::handleObjectEntry); stack.pop(); } @@ -296,18 +286,24 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private Object getValueFromInspector(Inspector inspector) { switch (inspector.type()) { - case STRING: + case STRING -> { return inspector.asString(); - case LONG: + } + case LONG -> { return String.valueOf(inspector.asLong()); - case DOUBLE: + } + case DOUBLE -> { return String.valueOf(inspector.asDouble()); - case NIX: + } + case NIX -> { return null; - case BOOL: + } + case BOOL -> { return String.valueOf(inspector.asBool()); - case DATA: + } + case DATA -> { return String.valueOf(inspector.asData()); + } } throw new IllegalArgumentException("Unhandled type " + inspector.type()); } @@ -372,7 +368,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return name.substring(0, 1).toUpperCase() + name.substring(1); } - private Constructor<?> lookupBuilderForStruct(String structName, String name, Class<?> currentClass) { + private Constructor<?> lookupBuilderForStruct(String structName, Class<?> currentClass) { String currentClassName = currentClass.getName(); Class<?> structClass = getInnerClass(currentClass, currentClassName + "$" + structName); if (structClass == null) { @@ -420,7 +416,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { String key = constructorCacheKey(structName, name, currentClass); Constructor<?> constructor = constructorCache.get(key); if (constructor == null) { - constructor = lookupBuilderForStruct(structName, name, currentClass); + constructor = lookupBuilderForStruct(structName, currentClass); if (constructor == null) return null; constructorCache.put(key, constructor); } @@ -437,7 +433,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private static class NamedBuilder { private final ConfigBuilder builder; - private final Stack<String> names = new Stack<>(); // if empty, the builder is the root builder + private final Deque<String> names = new ArrayDeque<>(); // if empty, the builder is the root builder NamedBuilder(ConfigBuilder builder) { this.builder = builder; @@ -456,7 +452,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { return names.peek(); } - Stack<String> nameStack() { + Deque<String> nameStack() { return names; } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java index 875033f1ffa..5193ecddc82 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java @@ -5,8 +5,9 @@ import com.yahoo.slime.Cursor; import com.yahoo.yolean.trace.TraceNode; import com.yahoo.yolean.trace.TraceVisitor; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.Iterator; -import java.util.Stack; /** * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}. @@ -17,7 +18,7 @@ public class SlimeTraceSerializer extends TraceVisitor { static final String TIMESTAMP = "timestamp"; static final String PAYLOAD = "payload"; static final String CHILDREN = "children"; - final Stack<Cursor> cursors = new Stack<>(); + final Deque<Cursor> cursors = new ArrayDeque<>(); public SlimeTraceSerializer(Cursor cursor) { cursors.push(cursor); diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java deleted file mode 100644 index ab07b669bd0..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config; - -import com.yahoo.foo.ArraytypesConfig; -import com.yahoo.foo.SimpletypesConfig; -import com.yahoo.foo.StructtypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.foo.MaptypesConfig; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.text.StringUtilities; -import com.yahoo.text.Utf8; -import org.junit.Test; -import org.junit.Ignore; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.StringReader; -import java.nio.charset.StandardCharsets; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class ConfigFileFormatterTest { - - private final String expected_simpletypes = "stringval \"foo\"\n" + - "intval 324234\n" + - "longval 324\n" + - "doubleval 3.455\n" + - "enumval VAL2\n" + - "boolval true\n"; - - @Test - public void require_that_basic_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("stringval", "foo"); - root.setString("intval", "324234"); - root.setString("longval", "324"); - root.setString("doubleval", "3.455"); - root.setString("enumval", "VAL2"); - root.setString("boolval", "true"); - - assertConfigFormat(slime, expected_simpletypes); - } - - @Test - public void require_that_basic_formatting_is_correct_with_types() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("stringval", "foo"); - root.setLong("intval", 324234); - root.setLong("longval", 324); - root.setDouble("doubleval", 3.455); - root.setString("enumval", "VAL2"); - root.setBool("boolval", true); - - assertConfigFormat(slime, expected_simpletypes); - } - - private void assertConfigFormat(Slime slime, String expected_simpletypes) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals(expected_simpletypes, baos.toString()); - } - - @Test - public void require_that_field_not_found_is_ignored() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("nosuchfield", "bar"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertTrue(baos.toString().isEmpty()); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_int_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("intval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_long_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("longval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_double_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("doubleval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - @Test - public void require_that_illegal_boolean_becomes_false() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("boolval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("boolval false\n", baos.toString()); - } - - // TODO: Remove this when we can reenable typechecking. - @Test - public void require_that_types_are_not_checked() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("enumval", "null"); - root.setString("intval", "null"); - root.setString("longval", "null"); - root.setString("boolval", "null"); - root.setString("doubleval", "null"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("enumval null\nintval null\nlongval null\nboolval false\ndoubleval null\n", - baos.toString(StandardCharsets.UTF_8)); - } - - // TODO: Reenable this when we can reenable typechecking. - @Ignore - @Test(expected = IllegalArgumentException.class) - public void require_that_illegal_enum_throws_exception() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("enumval", "invalid"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(new ByteArrayOutputStream(), slime); - } - - @Test - public void require_that_strings_are_encoded() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - String value = "\u7d22"; - root.setString("stringval", value); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("stringval \"" + value + "\"\n", baos.toString(StandardCharsets.UTF_8)); - } - - @Test - public void require_that_array_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor boolarr = root.setArray("boolarr"); - boolarr.addString("true"); - boolarr.addString("false"); - Cursor doublearr = root.setArray("doublearr"); - doublearr.addString("3.14"); - doublearr.addString("1.414"); - Cursor enumarr = root.setArray("enumarr"); - enumarr.addString("VAL1"); - enumarr.addString("VAL2"); - Cursor intarr = root.setArray("intarr"); - intarr.addString("3"); - intarr.addString("5"); - Cursor longarr = root.setArray("longarr"); - longarr.addString("55"); - longarr.addString("66"); - Cursor stringarr = root.setArray("stringarr"); - stringarr.addString("foo"); - stringarr.addString("bar"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("arraytypes", new StringReader(StringUtilities.implode(ArraytypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals( - "boolarr[0] true\n" + - "boolarr[1] false\n" + - "doublearr[0] 3.14\n" + - "doublearr[1] 1.414\n" + - "enumarr[0] VAL1\n" + - "enumarr[1] VAL2\n" + - "intarr[0] 3\n" + - "intarr[1] 5\n" + - "longarr[0] 55\n" + - "longarr[1] 66\n" + - "stringarr[0] \"foo\"\n" + - "stringarr[1] \"bar\"\n", - baos.toString()); - } - - @Test - public void require_that_map_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor boolval = root.setObject("boolmap"); - boolval.setString("foo", "true"); - boolval.setString("bar", "false"); - root.setObject("intmap").setString("foo", "1234"); - root.setObject("longmap").setString("foo", "12345"); - root.setObject("doublemap").setString("foo", "3.14"); - root.setObject("stringmap").setString("foo", "bar"); - root.setObject("innermap").setObject("bar").setString("foo", "1234"); - root.setObject("nestedmap").setObject("baz").setObject("inner").setString("foo", "1234"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("maptypes", new StringReader(StringUtilities.implode(MaptypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals( - "boolmap{\"foo\"} true\n" + - "boolmap{\"bar\"} false\n" + - "intmap{\"foo\"} 1234\n" + - "longmap{\"foo\"} 12345\n" + - "doublemap{\"foo\"} 3.14\n" + - "stringmap{\"foo\"} \"bar\"\n" + - "innermap{\"bar\"}.foo 1234\n" + - "nestedmap{\"baz\"}.inner{\"foo\"} 1234\n", - baos.toString()); - } - - @Test - public void require_that_struct_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor simple = root.setObject("simple"); - simple.setString("name", "myname"); - simple.setString("gender", "FEMALE"); - Cursor array = simple.setArray("emails"); - array.addString("foo@bar.com"); - array.addString("bar@baz.net"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals( - "simple.name \"myname\"\n" + - "simple.gender FEMALE\n" + - "simple.emails[0] \"foo@bar.com\"\n" + - "simple.emails[1] \"bar@baz.net\"\n", - baos.toString()); - } - - @Test - public void require_that_complex_struct_formatting_is_correct() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - Cursor nested = root.setObject("nested"); - Cursor nested_inner = nested.setObject("inner"); - nested_inner.setString("name", "baz"); - nested_inner.setString("gender", "FEMALE"); - Cursor nested_inner_arr = nested_inner.setArray("emails"); - nested_inner_arr.addString("foo"); - nested_inner_arr.addString("bar"); - - Cursor nestedarr = root.setArray("nestedarr"); - Cursor nestedarr1 = nestedarr.addObject(); - Cursor inner1 = nestedarr1.setObject("inner"); - inner1.setString("name", "foo"); - inner1.setString("gender", "FEMALE"); - Cursor inner1arr = inner1.setArray("emails"); - inner1arr.addString("foo@bar"); - inner1arr.addString("bar@foo"); - - Cursor complexarr = root.setArray("complexarr"); - Cursor complexarr1 = complexarr.addObject(); - Cursor innerarr1 = complexarr1.setArray("innerarr"); - Cursor innerarr11 = innerarr1.addObject(); - innerarr11.setString("name", "bar"); - innerarr11.setString("gender", "MALE"); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - InnerCNode def = new DefParser("structtypes", new StringReader(StringUtilities.implode(StructtypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("nested.inner.name \"baz\"\n" + - "nested.inner.gender FEMALE\n" + - "nested.inner.emails[0] \"foo\"\n" + - "nested.inner.emails[1] \"bar\"\n" + - "nestedarr[0].inner.name \"foo\"\n" + - "nestedarr[0].inner.gender FEMALE\n" + - "nestedarr[0].inner.emails[0] \"foo@bar\"\n" + - "nestedarr[0].inner.emails[1] \"bar@foo\"\n" + - "complexarr[0].innerarr[0].name \"bar\"\n" + - "complexarr[0].innerarr[0].gender MALE\n", - baos.toString()); - } - - @Test - public void require_that_strings_are_properly_escaped() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("stringval", "some\"quotes\\\"instring"); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - assertEquals("stringval \"some\\\"quotes\\\\\\\"instring\"\n", baos.toString()); - } - - @Test - @Ignore - public void require_that_utf8_works() throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - final String input = "Hei \u00E6\u00F8\u00E5 \n \uBC14\uB451 \u00C6\u00D8\u00C5 hallo"; - root.setString("stringval", input); - System.out.println(bytesToHexString(Utf8.toBytes(input))); - InnerCNode def = new DefParser("simpletypes", new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))).getTree(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new ConfigFileFormat(def).encode(baos, slime); - System.out.println(bytesToHexString(baos.toByteArray())); - assertEquals(Utf8.toString(baos.toByteArray()), "stringval \"" + input + "\"\n"); - } - - private static String bytesToHexString(byte[] bytes){ - StringBuilder sb = new StringBuilder(); - for(byte b : bytes){ - sb.append(String.format("%02x", b&0xff)); - } - return sb.toString(); - } -} diff --git a/configserver/pom.xml b/configserver/pom.xml index e05e1f5e5ba..ef0fde3e57e 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -32,10 +32,6 @@ <groupId>jakarta.xml.bind</groupId> <artifactId>jakarta.xml.bind-api</artifactId> </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> </exclusions> </dependency> <dependency> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java index 58d2e6a2c90..47bfc3662b5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java @@ -33,7 +33,7 @@ public class SecretStoreValidator { private static final String PROTOCOL = "http://"; private static final String AWS_PARAMETER_VALIDATION_HANDLER_POSTFIX = ":4080/validate-secret-store"; private final SecretStore secretStore; - private final CloseableHttpClient httpClient = VespaHttpClientBuilder.custom().buildClient(); + private final CloseableHttpClient httpClient = VespaHttpClientBuilder.custom().buildClient();; public SecretStoreValidator(SecretStore secretStore) { this.secretStore = secretStore; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 9821cbe9c15..a7f4ef5d513 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -24,8 +24,9 @@ public class SimpleHttpFetcher implements HttpFetcher { private final CloseableHttpClient client; - public SimpleHttpFetcher(Duration connectTimeout) { this(connectTimeout, null); } - + public SimpleHttpFetcher(Duration connectTimeout) { + this(connectTimeout, null); + } public SimpleHttpFetcher(Duration connectTimeout, NodeHostnameVerifier verifier) { VespaHttpClientBuilder builder = VespaHttpClientBuilder.custom().connectTimeout(Timeout.of(connectTimeout)); if (verifier != null) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index c9a03e362ee..61c0c17264c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -126,31 +126,22 @@ public class ClusterDeploymentMetricsRetriever { Supplier<DeploymentMetricsAggregator> aggregator = () -> clusterMetricsMap.computeIfAbsent(clusterInfo, c -> new DeploymentMetricsAggregator()); switch (serviceName) { - case VESPA_CONTAINER: + case VESPA_CONTAINER -> { optionalDouble(values.field("query_latency.sum")).ifPresent(qlSum -> - aggregator.get() - .addContainerLatency(qlSum, values.field("query_latency.count").asDouble())); + aggregator.get().addContainerLatency(qlSum, values.field("query_latency.count").asDouble())); optionalDouble(values.field("feed.latency.sum")).ifPresent(flSum -> - aggregator.get() - .addFeedLatency(flSum, values.field("feed.latency.count").asDouble())); - break; - case VESPA_QRSERVER: - optionalDouble(values.field("query_latency.sum")).ifPresent(qlSum -> - aggregator.get() - .addQrLatency(qlSum, values.field("query_latency.count").asDouble())); - break; - case VESPA_DISTRIBUTOR: - optionalDouble(values.field("vds.distributor.docsstored.average")) - .ifPresent(docCount -> aggregator.get().addDocumentCount(docCount)); - break; - case VESPA_CONTAINER_CLUSTERCONTROLLER: - optionalDouble(values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_MEMORY_UTILIZATION.max())).ifPresent(memoryUtil -> - aggregator.get() - .addMemoryUsage(memoryUtil, - values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MEMORY_LIMIT.last()).asDouble()) - .addDiskUsage(values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_DISK_UTILIZATION.max()).asDouble(), - values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_DISK_LIMIT.last()).asDouble())); - break; + aggregator.get().addFeedLatency(flSum, values.field("feed.latency.count").asDouble())); + } + case VESPA_QRSERVER -> optionalDouble(values.field("query_latency.sum")).ifPresent(qlSum -> + aggregator.get().addQrLatency(qlSum, values.field("query_latency.count").asDouble())); + case VESPA_DISTRIBUTOR -> optionalDouble(values.field("vds.distributor.docsstored.average")) + .ifPresent(docCount -> aggregator.get().addDocumentCount(docCount)); + case VESPA_CONTAINER_CLUSTERCONTROLLER -> + optionalDouble(values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_MEMORY_UTILIZATION.max())).ifPresent(memoryUtil -> + aggregator.get() + .addMemoryUsage(memoryUtil, values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MEMORY_LIMIT.last()).asDouble()) + .addDiskUsage(values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_MAX_DISK_UTILIZATION.max()).asDouble(), + values.field(ContainerMetrics.CLUSTER_CONTROLLER_RESOURCE_USAGE_DISK_LIMIT.last()).asDouble())); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java index 5f3711e43b6..c137b80e951 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterProtonMetricsRetriever.java @@ -26,15 +26,12 @@ public class ClusterProtonMetricsRetriever { private static final Logger log = Logger.getLogger(ClusterProtonMetricsRetriever.class.getName()); - private static final CloseableHttpClient httpClient = - VespaHttpClientBuilder - .custom() - .connectTimeout(Timeout.ofSeconds(10)) - .apacheBuilder() - .setDefaultRequestConfig(RequestConfig.custom() - .setResponseTimeout(Timeout.ofSeconds(10)) - .build()) - .build(); + private static final CloseableHttpClient httpClient = VespaHttpClientBuilder + .custom() + .connectTimeout(Timeout.ofSeconds(10)) + .apacheBuilder() + .setDefaultRequestConfig(RequestConfig.custom().setResponseTimeout(Timeout.ofSeconds(10)).build()) + .build(); public Map<String, ProtonMetricsAggregator> requestMetricsGroupedByCluster(Collection<URI> hosts) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java b/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java index 60a9dbc96db..4378c4cbd47 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java @@ -5,9 +5,10 @@ import com.google.common.collect.ContiguousSet; import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Range; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.HashSet; import java.util.Set; -import java.util.Stack; /** * Allocates port ranges for all configserver tests. @@ -28,7 +29,7 @@ public class PortRangeAllocator { private static class PortRange { private final Set<Integer> takenPorts = new HashSet<>(); - private final Stack<Integer> freePorts = new Stack<>(); + private final Deque<Integer> freePorts = new ArrayDeque<>(); private static final int first = 18651; private static final int last = 18899; // see: factory/doc/port-ranges diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java index f4db1c5085a..f7aea8abbd1 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -52,7 +52,7 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext private final Executor renderingExecutor; - private ChainRegistry<COMPONENT> chainRegistry; + private final ChainRegistry<COMPONENT> chainRegistry; private final ComponentRegistry<Renderer> renderers; @@ -109,7 +109,6 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext } @Override - @SuppressWarnings("unchecked") public HttpResponse handle(HttpRequest request, ContentChannel channel) { com.yahoo.processing.Request processingRequest = new com.yahoo.processing.Request(); populate("", request.propertyMap(), processingRequest.properties()); diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java index a91a200f9cc..23f89962752 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java @@ -5,7 +5,6 @@ import org.apache.hc.client5.http.SystemDefaultDnsResolver; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.classic.methods.HttpUriRequest; -import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.entity.GzipCompressingEntity; import org.apache.hc.client5.http.entity.mime.FormBodyPart; import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; @@ -72,7 +71,6 @@ public class SimpleHttpClient implements AutoCloseable { new DefaultHostnameVerifier()); PoolingHttpClientConnectionManager connManager = PoolingHttpClientConnectionManagerBuilder.create() .setSSLSocketFactory(sslConnectionFactory) - .setDefaultConnectionConfig(ConnectionConfig.custom().build()) .setDnsResolver(new SystemDefaultDnsResolver() { @Override public InetAddress[] resolve(String host) throws UnknownHostException { @@ -192,18 +190,16 @@ public class SimpleHttpClient implements AutoCloseable { return this; } - public ResponseValidator expectHeader(String headerName, Matcher<String> matcher) { + public void expectHeader(String headerName, Matcher<String> matcher) { Header firstHeader = response.getFirstHeader(headerName); String headerValue = firstHeader != null ? firstHeader.getValue() : null; assertThat(headerValue, matcher); assertNotNull(firstHeader); - return this; } - public ResponseValidator expectNoHeader(String headerName) { + public void expectNoHeader(String headerName) { Header firstHeader = response.getFirstHeader(headerName); assertThat(firstHeader, is(nullValue())); - return this; } public ResponseValidator expectContent(final Matcher<String> matcher) { diff --git a/container-core/src/test/java/com/yahoo/processing/test/ProcessorLibrary.java b/container-core/src/test/java/com/yahoo/processing/test/ProcessorLibrary.java index ee8dbd8dccb..5f5b7171923 100644 --- a/container-core/src/test/java/com/yahoo/processing/test/ProcessorLibrary.java +++ b/container-core/src/test/java/com/yahoo/processing/test/ProcessorLibrary.java @@ -10,9 +10,18 @@ import com.yahoo.processing.execution.Execution; import com.yahoo.processing.execution.ExecutionWithResponse; import com.yahoo.processing.execution.RunnableExecution; import com.yahoo.processing.request.ErrorMessage; -import com.yahoo.processing.response.*; - -import java.util.*; +import com.yahoo.processing.response.AbstractData; +import com.yahoo.processing.response.ArrayDataList; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; +import com.yahoo.processing.response.FutureResponse; +import com.yahoo.processing.response.IncomingData; +import com.yahoo.processing.response.Ordered; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -50,7 +59,7 @@ public class ProcessorLibrary { public static class MapData extends AbstractData { - private Map map = new LinkedHashMap(); + private final Map map = new LinkedHashMap(); public MapData(Request request) { super(request); @@ -157,7 +166,7 @@ public class ProcessorLibrary { @SafeVarargs @SuppressWarnings("varargs") public Federator(boolean ordered, Chain<? extends Processor>... chains) { - this.chains = Arrays.asList(chains); + this.chains = List.of(chains); this.ordered = ordered; } @@ -206,7 +215,7 @@ public class ProcessorLibrary { @SafeVarargs @SuppressWarnings("varargs") public EagerReturnFederator(boolean ordered, Chain<? extends Processor>... chains) { - this.chains = Arrays.asList(chains); + this.chains = List.of(chains); this.ordered = ordered; } @@ -312,21 +321,6 @@ public class ProcessorLibrary { } - /** Allows waiting for that request to happen. */ - public static class RequestCounter extends Processor { - - /** The incoming data this has created */ - public final CompletableFuture<IncomingData> incomingData = new CompletableFuture<>(); - - @Override - public Response process(Request request, Execution execution) { - ArrayDataList dataList = ArrayDataList.createAsync(request); - incomingData.complete(dataList.incoming()); - return new Response(dataList); - } - - } - /** * Multiples the amount of data returned by parallelism by performing parallel executions of the rest of the chain */ @@ -413,30 +407,11 @@ public class ProcessorLibrary { } /** - * A processor which on invocation prints the string given on construction - */ - public static class Echo extends Processor { - - private String s; - - public Echo(String s) { - this.s = s; - } - - @Override - public Response process(Request request, Execution execution) { - System.out.println(s); - return execution.process(request); - } - - } - - /** * A processor which adds a StringData item containing the string given in the constructor to every response */ public static class StringDataAdder extends Processor { - private String string; + private final String string; public StringDataAdder(String string) { this.string = string; @@ -458,7 +433,7 @@ public class ProcessorLibrary { */ public static class ErrorAdder extends Processor { - private ErrorMessage errorMessage; + private final ErrorMessage errorMessage; public ErrorAdder(ErrorMessage errorMessage) { this.errorMessage = errorMessage; @@ -478,7 +453,7 @@ public class ProcessorLibrary { */ public static class StringDataListAdder extends Processor { - private String[] strings; + private final String[] strings; public StringDataListAdder(String... strings) { this.strings = strings; @@ -502,8 +477,8 @@ public class ProcessorLibrary { */ public static class Trace extends Processor { - private String traceMessage; - private int traceLevel; + private final String traceMessage; + private final int traceLevel; public Trace(String traceMessage, int traceLevel) { this.traceMessage = traceMessage; diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index ae4d0e713f8..785475ae77e 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -8185,6 +8185,7 @@ ], "methods" : [ "public java.lang.String name()", + "public com.yahoo.search.schema.Schema schema()", "public boolean hasSummaryFeatures()", "public boolean hasRankFeatures()", "public java.util.Map inputs()", diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java index f0a18954e40..98a8e5af00d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java @@ -3,7 +3,11 @@ package com.yahoo.prelude.querytransform; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.prelude.query.parser.AnyParser; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -14,10 +18,10 @@ import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchchain.PhaseNames; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.Iterator; -import java.util.Stack; import static com.yahoo.prelude.querytransform.NormalizingSearcher.ACCENT_REMOVAL; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @@ -76,15 +80,14 @@ public class RecallSearcher extends Searcher { * @return True if a rank item was found. */ private static boolean hasRankItem(Item root) { - Stack<Item> stack = new Stack<>(); + Deque<Item> stack = new ArrayDeque<>(); stack.push(root); while (!stack.isEmpty()) { Item item = stack.pop(); if (item instanceof RankItem) { return true; } - if (item instanceof CompositeItem) { - CompositeItem lst = (CompositeItem)item; + if (item instanceof CompositeItem lst) { for (Iterator<Item> it = lst.getItemIterator(); it.hasNext();) { stack.push(it.next()); } @@ -102,20 +105,18 @@ public class RecallSearcher extends Searcher { * @return The first node found. */ private static WordItem findOrigWordItem(Item root, String value) { - Stack<Item> stack = new Stack<>(); + Deque<Item> stack = new ArrayDeque<>(); stack.push(root); while (!stack.isEmpty()) { Item item = stack.pop(); if (item.getCreator() == Item.ItemCreator.ORIG && - item instanceof WordItem) + item instanceof WordItem word) { - WordItem word = (WordItem)item; if (word.getWord().equals(value)) { return word; } } - if (item instanceof CompositeItem) { - CompositeItem lst = (CompositeItem)item; + if (item instanceof CompositeItem lst) { for (Iterator<Item> it = lst.getItemIterator(); it.hasNext();) { stack.push(it.next()); } @@ -130,15 +131,14 @@ public class RecallSearcher extends Searcher { * @param root The root of the tree to update. */ private static void updateFilterTerms(Item root) { - Stack<Item> stack = new Stack<>(); + Deque<Item> stack = new ArrayDeque<>(); stack.push(root); while (!stack.isEmpty()) { Item item = stack.pop(); if (item.getCreator() == Item.ItemCreator.FILTER) { item.setRanked(false); } - if (item instanceof CompositeItem) { - CompositeItem lst = (CompositeItem)item; + if (item instanceof CompositeItem lst) { for (Iterator<Item> it = lst.getItemIterator(); it.hasNext();) { stack.push(it.next()); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 1da16b4d277..efb02034db9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -7,7 +7,14 @@ import com.yahoo.prelude.query.TermType; import com.yahoo.prelude.semantics.rule.Condition; import com.yahoo.prelude.semantics.rule.ProductionRule; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + /** * A particular evaluation of a particular rule. @@ -41,15 +48,15 @@ public class RuleEvaluation { private String currentContext; /** A list of referencedMatches */ - private final List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); + private final List<ReferencedMatches> referencedMatchesList = new ArrayList<>(); - private final List<Match> nonreferencedMatches = new java.util.ArrayList<>(); + private final List<Match> nonreferencedMatches = new ArrayList<>(); /** The evaluation owning this */ private final Evaluation evaluation; /** The choice points saved in this evaluation */ - private Stack<Choicepoint> choicepoints = null; + private Deque<Choicepoint> choicepoints = null; /* The last value returned by a condition evaluated in this, may be null */ private Object value = null; @@ -98,20 +105,18 @@ public class RuleEvaluation { int calculateMatchDigest(ProductionRule rule) { int matchDigest = rule.hashCode(); int matchCounter = 1; - for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches = i.next(); + for (ReferencedMatches matches : referencedMatchesList) { int termCounter = 0; for (Iterator<Match> j = matches.matchIterator(); j.hasNext(); ) { Match match = j.next(); - matchDigest = 7 * matchDigest * matchCounter+ - 71 * termCounter + - match.hashCode(); + matchDigest = 7 * matchDigest * matchCounter + + 71 * termCounter + + match.hashCode(); termCounter++; } matchCounter++; } - for (Iterator<Match> i = nonreferencedMatches.iterator(); i.hasNext(); ) { - Match match = i.next(); + for (Match match : nonreferencedMatches) { matchDigest = 7 * matchDigest * matchCounter + match.hashCode(); matchCounter++; } @@ -163,10 +168,8 @@ public class RuleEvaluation { public int getPosition() { return position; } /** Sets a new current label and returns the previous one */ - public String setCurrentLabel(String currentLabel) { - String oldLabel = currentLabel; + public void setCurrentLabel(String currentLabel) { this.currentLabel = currentLabel; - return oldLabel; } public String getCurrentLabel() { return currentLabel; } @@ -226,8 +229,7 @@ public class RuleEvaluation { /** Returns the referenced matches for a context name, or null if none */ public ReferencedMatches getReferencedMatches(String name) { - for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches = i.next(); + for (ReferencedMatches matches : referencedMatchesList) { if (name.equals(matches.getContextName())) return matches; } @@ -307,7 +309,7 @@ public class RuleEvaluation { public Choicepoint getChoicepoint(Condition condition, boolean create) { if (choicepoints == null) { if ( ! create) return null; - choicepoints = new java.util.Stack<>(); + choicepoints = new ArrayDeque<>(); } Choicepoint choicepoint=lookupChoicepoint(condition); if (choicepoint == null) { @@ -319,8 +321,7 @@ public class RuleEvaluation { } private Choicepoint lookupChoicepoint(Condition condition) { - for (Iterator<Choicepoint> i = choicepoints.iterator(); i.hasNext(); ) { - Choicepoint choicepoint = i.next(); + for (Choicepoint choicepoint : choicepoints) { if (condition == choicepoint.getCondition()) return choicepoint; } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java index 98dd2ee3c88..fd3651e6787 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java @@ -1,9 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.LinkedList; import java.util.List; -import java.util.Stack; /** * This is a helper class for resolving arithmetic operations over {@link GroupingExpression} objects. To resolve an @@ -25,7 +26,7 @@ public class MathResolver { private final int pre; private final String image; - private Type(int pre, String image) { + Type(int pre, String image) { this.pre = pre; this.image = image; } @@ -57,7 +58,7 @@ public class MathResolver { if (items.size() == 1) { return items.remove(0).exp; // optimize common case } - Stack<Item> stack = new Stack<>(); + Deque<Item> stack = new ArrayDeque<>(); stack.push(items.remove(0)); while (!items.isEmpty()) { Item item = items.remove(0); @@ -69,30 +70,19 @@ public class MathResolver { while (stack.size() > 1) { pop(stack); } - return stack.remove(0).exp; + return stack.pop().exp; } - private void pop(Stack<Item> stack) { + private void pop(Deque<Item> stack) { Item rhs = stack.pop(); Item lhs = stack.peek(); switch (rhs.type) { - case ADD: - lhs.exp = new AddFunction(lhs.exp, rhs.exp); - break; - case DIV: - lhs.exp = new DivFunction(lhs.exp, rhs.exp); - break; - case MOD: - lhs.exp = new ModFunction(lhs.exp, rhs.exp); - break; - case MUL: - lhs.exp = new MulFunction(lhs.exp, rhs.exp); - break; - case SUB: - lhs.exp = new SubFunction(lhs.exp, rhs.exp); - break; - default: - throw new UnsupportedOperationException("Operator " + rhs.type + " not supported."); + case ADD -> lhs.exp = new AddFunction(lhs.exp, rhs.exp); + case DIV -> lhs.exp = new DivFunction(lhs.exp, rhs.exp); + case MOD -> lhs.exp = new ModFunction(lhs.exp, rhs.exp); + case MUL -> lhs.exp = new MulFunction(lhs.exp, rhs.exp); + case SUB -> lhs.exp = new SubFunction(lhs.exp, rhs.exp); + default -> throw new UnsupportedOperationException("Operator " + rhs.type + " not supported."); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java index e5b45373370..958f343fac5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java @@ -18,11 +18,13 @@ import com.yahoo.searchlib.aggregation.HitsAggregationResult; import com.yahoo.searchlib.expression.ExpressionNode; import com.yahoo.searchlib.expression.RangeBucketPreDefFunctionNode; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import java.util.OptionalLong; -import java.util.Stack; +import java.util.Deque; import java.util.TimeZone; /** @@ -61,7 +63,7 @@ class RequestBuilder { * @return This, to allow chaining. */ public RequestBuilder setRootOperation(GroupingOperation root) { - root.getClass(); // throws NullPointerException + Objects.requireNonNull(root, "Root must be non-null"); this.root = root; return this; } @@ -125,7 +127,7 @@ class RequestBuilder { Grouping grouping = new Grouping(); grouping.getRoot().setTag(++tag); grouping.setForceSinglePass(root.getForceSinglePass() || root.containsHint("singlepass")); - Stack<BuildFrame> stack = new Stack<>(); + Deque<BuildFrame> stack = new ArrayDeque<>(); stack.push(new BuildFrame(grouping, new BuildState(), root)); while (!stack.isEmpty()) { BuildFrame frame = stack.pop(); @@ -317,11 +319,10 @@ class RequestBuilder { result.setTag(++tag); String label = exp.getLabel(); - if (result instanceof HitsAggregationResult) { + if (result instanceof HitsAggregationResult hits) { if (label != null) { throw new UnsupportedOperationException("Can not label expression '" + exp + "'."); } - HitsAggregationResult hits = (HitsAggregationResult)result; if (frame.state.max != null) { transform.putMax(tag, frame.state.max, "hit list"); int offset = transform.getOffset(tag); diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index 39d4a389e6f..85bb3915975 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -21,6 +21,9 @@ public class RankProfile { private final boolean hasRankFeatures; private final Map<String, TensorType> inputs; + // Assigned when this is added to a schema + private Schema schema = null; + private RankProfile(Builder builder) { this.name = builder.name; this.hasSummaryFeatures = builder.hasSummaryFeatures; @@ -30,6 +33,16 @@ public class RankProfile { public String name() { return name; } + /** Returns the schema owning this, or null if this is not added to a schema */ + public Schema schema() { return schema; } + + void setSchema(Schema schema) { + if ( this.schema != null) + throw new IllegalStateException("Cannot add rank profile '" + name + "' to schema '" + schema.name() + + "' as it is already added to schema '" + this.schema.name() + "'"); + this.schema = schema; + } + /** Returns true if this rank profile has summary features. */ public boolean hasSummaryFeatures() { return hasSummaryFeatures; } @@ -42,8 +55,7 @@ public class RankProfile { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof RankProfile)) return false; - RankProfile other = (RankProfile)o; + if ( ! (o instanceof RankProfile other)) return false; if ( ! other.name.equals(this.name)) return false; if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; if ( other.hasRankFeatures != this.hasRankFeatures) return false; @@ -58,7 +70,7 @@ public class RankProfile { @Override public String toString() { - return "rank profile '" + name + "'"; + return "rank profile '" + name + "'" + (schema == null ? "" : " in " + schema); } public static class Builder { diff --git a/container-search/src/main/java/com/yahoo/search/schema/Schema.java b/container-search/src/main/java/com/yahoo/search/schema/Schema.java index 2ab5a30fbd7..c20aa1e81bd 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/Schema.java +++ b/container-search/src/main/java/com/yahoo/search/schema/Schema.java @@ -27,6 +27,7 @@ public class Schema { this.name = builder.name; this.rankProfiles = Collections.unmodifiableMap(builder.rankProfiles); this.documentSummaries = Collections.unmodifiableMap(builder.documentSummaries); + rankProfiles.values().forEach(rankProfile -> rankProfile.setSchema(this)); } public String name() { return name; } @@ -36,8 +37,7 @@ public class Schema { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof Schema)) return false; - Schema other = (Schema)o; + if ( ! (o instanceof Schema other)) return false; if ( ! other.name.equals(this.name)) return false; if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; if ( ! other.documentSummaries.equals(this.documentSummaries)) return false; diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java index 6482070bc03..d29964ea9c5 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfo.java @@ -9,6 +9,7 @@ import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.tensor.TensorType; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -17,6 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; /** * Information about all the schemas configured in the application this container is a part of. @@ -124,6 +126,13 @@ public class SchemaInfo { return schemas.stream().filter(schema -> names.contains(schema.name())).toList(); } + private List<RankProfile> profilesNamed(String name) { + return schemas.stream() + .filter(schema -> schema.rankProfiles().containsKey(name)) + .map(schema -> schema.rankProfiles().get(name)) + .toList(); + } + /** * Returns the type of the given rank feature name in the given profile, * if it can be uniquely determined. @@ -131,23 +140,27 @@ public class SchemaInfo { * @param rankFeature the rank feature name, a string on the form "query(name)" * @param rankProfile the name of the rank profile in which to locate the input declaration * @return the type of the declared input, or null if it is not declared or the rank profile is not found - * @throws IllegalArgumentException if the feature is declared in this rank profile in multiple schemas + * @throws IllegalArgumentException if the given rank profile does not exist in any schema, or the + * feature is declared in this rank profile in multiple schemas * of this session with conflicting types */ public TensorType rankProfileInput(String rankFeature, String rankProfile) { + if (schemas.isEmpty()) return null; // no matching schemas - validated elsewhere + List<RankProfile> profiles = profilesNamed(rankProfile); + if (profiles.isEmpty()) + throw new IllegalArgumentException("No profile named '" + rankProfile + "' exists in schemas [" + + schemas.stream().map(Schema::name).collect(Collectors.joining(", ")) + "]"); TensorType foundType = null; - Schema declaringSchema = null; - for (Schema schema : schemas) { - RankProfile profile = schema.rankProfiles().get(rankProfile); - if (profile == null) continue; + RankProfile declaringProfile = null; + for (RankProfile profile : profiles) { TensorType newlyFoundType = profile.inputs().get(rankFeature); if (newlyFoundType == null) continue; if (foundType != null && ! newlyFoundType.equals(foundType)) throw new IllegalArgumentException("Conflicting input type declarations for '" + rankFeature + "': " + - "Declared as " + foundType + " in " + profile + " in " + declaringSchema + - ", and as " + newlyFoundType + " in " + profile + " in " + schema); + "Declared as " + foundType + " in " + declaringProfile + + ", and as " + newlyFoundType + " in " + profile); foundType = newlyFoundType; - declaringSchema = schema; + declaringProfile = profile; } return foundType; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index 2c5c1ad83fe..0d0a4c3b695 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -74,7 +74,7 @@ import java.util.stream.Stream; */ final class ProgramParser { - public yqlplusParser prepareParser(String programName, String input) throws IOException { + public yqlplusParser prepareParser(String programName, String input) { return prepareParser(programName, new CaseInsensitiveCharStream(CharStreams.fromString(input))); } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java index e2d1555701c..a879f142bcc 100755 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/RecallSearcherTestCase.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform.test; +import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Deque; import java.util.Iterator; import java.util.List; -import java.util.Stack; -import com.yahoo.prelude.IndexFacts; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.prelude.query.CompositeItem; @@ -45,13 +44,13 @@ public class RecallSearcherTestCase { @Test void testParse() { List<String> empty = new ArrayList<>(); - assertQueryTree("?query=foo", Arrays.asList("foo"), empty); - assertQueryTree("?recall=%2bfoo", empty, Arrays.asList("foo")); - assertQueryTree("?query=foo&filter=bar&recall=%2bbaz", Arrays.asList("foo", "bar"), Arrays.asList("baz")); - assertQueryTree("?query=foo+bar&filter=baz&recall=%2bcox", Arrays.asList("foo", "bar", "baz"), Arrays.asList("cox")); - assertQueryTree("?query=foo&filter=bar+baz&recall=%2bcox", Arrays.asList("foo", "bar", "baz"), Arrays.asList("cox")); - assertQueryTree("?query=foo&filter=bar&recall=-baz+%2bcox", Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox")); - assertQueryTree("?query=foo%20bar&recall=%2bbaz%20-cox", Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox")); + assertQueryTree("?query=foo", List.of("foo"), empty); + assertQueryTree("?recall=%2bfoo", empty, List.of("foo")); + assertQueryTree("?query=foo&filter=bar&recall=%2bbaz", List.of("foo", "bar"), List.of("baz")); + assertQueryTree("?query=foo+bar&filter=baz&recall=%2bcox", List.of("foo", "bar", "baz"), List.of("cox")); + assertQueryTree("?query=foo&filter=bar+baz&recall=%2bcox", List.of("foo", "bar", "baz"), List.of("cox")); + assertQueryTree("?query=foo&filter=bar&recall=-baz+%2bcox", List.of("foo", "bar"), List.of("baz", "cox")); + assertQueryTree("?query=foo%20bar&recall=%2bbaz%20-cox", List.of("foo", "bar"), List.of("baz", "cox")); } private static void assertQueryTree(String query, List<String> ranked, List<String> unranked) { @@ -65,7 +64,7 @@ public class RecallSearcherTestCase { List<String> myRanked = new ArrayList<>(ranked); List<String> myUnranked = new ArrayList<>(unranked); - Stack<Item> stack = new Stack<>(); + Deque<Item> stack = new ArrayDeque<>(); stack.push(obj.getModel().getQueryTree().getRoot()); while (!stack.isEmpty()) { Item item = stack.pop(); @@ -85,8 +84,7 @@ public class RecallSearcherTestCase { myUnranked.remove(idx); } } - if (item instanceof CompositeItem) { - CompositeItem lst = (CompositeItem)item; + if (item instanceof CompositeItem lst) { for (Iterator<?> it = lst.getItemIterator(); it.hasNext();) { stack.push((Item)it.next()); } diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index da1d09fec1e..cbe4ddcbc63 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -50,10 +50,12 @@ public class RankProfileInputTest { assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); } - { // Resolution is limited to the correct sources - Query query = createTensor1Query(tensorString, "bOnly", "sources=a"); - assertEquals(0, query.errors().size()); - assertEquals(tensorString, query.properties().get("ranking.features.query(myTensor1)"), "Not converted to tensor"); + try { + createTensor1Query(tensorString, "bOnly", "sources=a"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("No profile named 'bOnly' exists in schemas [a]", Exceptions.toMessageString(e)); } } @@ -237,20 +239,19 @@ public class RankProfileInputTest { private SchemaInfo createSchemaInfo() { List<Schema> schemas = new ArrayList<>(); - RankProfile common = new RankProfile.Builder("commonProfile") + RankProfile.Builder common = new RankProfile.Builder("commonProfile") .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .addInput("query(myTensor2)", TensorType.fromSpec("tensor(x[2],y[2])")) .addInput("query(myTensor3)", TensorType.fromSpec("tensor(x[2],y[2])")) - .addInput("query(myTensor4)", TensorType.fromSpec("tensor<float>(x[5])")) - .build(); + .addInput("query(myTensor4)", TensorType.fromSpec("tensor<float>(x[5])")); schemas.add(new Schema.Builder("a") - .add(common) + .add(common.build()) .add(new RankProfile.Builder("inconsistent") .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .build()) .build()); schemas.add(new Schema.Builder("b") - .add(common) + .add(common.build()) .add(new RankProfile.Builder("inconsistent") .addInput("query(myTensor1)", TensorType.fromSpec("tensor(x[10])")) .build()) diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg index ec091ecf2ea..cf17bc6ffcf 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg @@ -1,16 +1,14 @@ queryprofile[4] queryprofile[0].id "default" -queryprofile[0].property[5] +queryprofile[0].property[4] queryprofile[0].property[0].name "hits" queryprofile[0].property[0].value "5" queryprofile[0].property[1].name "model.defaultIndex" queryprofile[0].property[1].value "title" queryprofile[0].property[2].name "ranking.features.query(scorelimit)" queryprofile[0].property[2].value "-20" -queryprofile[0].property[3].name "ranking.profile" -queryprofile[0].property[3].value "production1" -queryprofile[0].property[4].name "ranking.properties.dotProduct.X" -queryprofile[0].property[4].value "(a:1,b:2)" +queryprofile[0].property[3].name "ranking.properties.dotProduct.X" +queryprofile[0].property[3].value "(a:1,b:2)" queryprofile[1].id "multi" queryprofile[1].inherit[1] queryprofile[1].inherit[0] "default" diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java index 40d6f19c275..ba0c3f87900 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTest.java @@ -2,9 +2,11 @@ package com.yahoo.search.schema; import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author bratseth @@ -40,10 +42,17 @@ public class SchemaInfoTest { "a", "", "inconsistent", "query(myTensor1)"); tester.assertInput(TensorType.fromSpec("tensor(x[10])"), "b", "", "inconsistent", "query(myTensor1)"); - tester.assertInput(null, - "a", "", "bOnly", "query(myTensor1)"); tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), "ab", "", "bOnly", "query(myTensor1)"); + try { + tester.assertInput(null, + "a", "", "bOnly", "query(myTensor1)"); + fail("Expected exception since bOnly is not in a"); + } + catch (IllegalArgumentException e) { + assertEquals("No profile named 'bOnly' exists in schemas [a]", + Exceptions.toMessageString(e)); + } } } diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index 4e7dc27e73c..a46f3480d50 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -58,14 +58,13 @@ public class SchemaInfoTester { static SchemaInfo createSchemaInfo() { List<Schema> schemas = new ArrayList<>(); - RankProfile common = new RankProfile.Builder("commonProfile") + RankProfile.Builder common = new RankProfile.Builder("commonProfile") .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .addInput("query(myTensor2)", TensorType.fromSpec("tensor(x[2],y[2])")) .addInput("query(myTensor3)", TensorType.fromSpec("tensor(x[2],y[2])")) - .addInput("query(myTensor4)", TensorType.fromSpec("tensor<float>(x[5])")) - .build(); + .addInput("query(myTensor4)", TensorType.fromSpec("tensor<float>(x[5])")); schemas.add(new Schema.Builder("a") - .add(common) + .add(common.build()) .add(new RankProfile.Builder("inconsistent") .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) .build()) @@ -76,7 +75,7 @@ public class SchemaInfoTester { .build()) .build()); schemas.add(new Schema.Builder("b") - .add(common) + .add(common.build()) .add(new RankProfile.Builder("inconsistent") .addInput("query(myTensor1)", TensorType.fromSpec("tensor(x[10])")) .build()) @@ -92,6 +91,7 @@ public class SchemaInfoTester { /** Creates the same schema info as createSchemaInfo from config objects. */ static SchemaInfo createSchemaInfoFromConfig() { + var indexInfoConfig = new IndexInfoConfig.Builder(); var rankProfileCommon = new SchemaInfoConfig.Schema.Rankprofile.Builder(); diff --git a/container-test/pom.xml b/container-test/pom.xml index d6c9ef666da..a0358e79a8f 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -23,10 +23,6 @@ <version>${project.version}</version> <exclusions> <exclusion> - <groupId>biz.aQute.bnd</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> </exclusion> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index c0989f61e36..14f2b38f24a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -280,7 +280,8 @@ public class InternalStepRunner implements StepRunner { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Validating CA signed certificate requested for app: not yet available"); + logger.log("Creating a CA signed certificate for the application. " + + "This may take up to " + timeouts.endpointCertificate() + " on first deployment."); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { logger.log(WARNING, "CA signed certificate for app not available within " + timeouts.endpointCertificate() + ": " + Exceptions.toMessageString(e)); diff --git a/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java b/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java index 775ce41d303..faae78ff8ee 100644 --- a/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java +++ b/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java @@ -1,9 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.annotation; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.ListIterator; import java.util.NoSuchElementException; -import java.util.Stack; /** * ListIterator implementation which performs a depth-first traversal of SpanNodes. @@ -11,11 +12,11 @@ import java.util.Stack; * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> */ class RecursiveNodeIterator implements ListIterator<SpanNode> { - protected Stack<PeekableListIterator<SpanNode>> stack = new Stack<PeekableListIterator<SpanNode>>(); + protected Deque<PeekableListIterator<SpanNode>> stack = new ArrayDeque<>(); protected ListIterator<SpanNode> iteratorFromLastCallToNext = null; RecursiveNodeIterator(ListIterator<SpanNode> it) { - stack.push(new PeekableListIterator<SpanNode>(it)); + stack.push(new PeekableListIterator<>(it)); } protected RecursiveNodeIterator() { @@ -38,7 +39,7 @@ class RecursiveNodeIterator implements ListIterator<SpanNode> { if (!iterator.traversed) { //we set the traversed flag on our way down iterator.traversed = true; - stack.push(new PeekableListIterator<SpanNode>(node.childIterator())); + stack.push(new PeekableListIterator<>(node.childIterator())); return hasNext(); } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index 790cc5d4cde..dba1c0783cf 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -140,9 +140,8 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof Array)) return false; + if (!(o instanceof Array a)) return false; if (!super.equals(o)) return false; - Array a = (Array) o; if (values.size() != a.values.size()) return false; if (values instanceof ListWrapper && !(a.values instanceof ListWrapper)) { return equalsWithListWrapper(a.values, (ListWrapper<? extends FieldValue>) values); diff --git a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java index b160293440e..76de41fc39e 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java @@ -8,9 +8,10 @@ import com.yahoo.document.select.Context; import com.yahoo.document.select.Result; import com.yahoo.document.select.Visitor; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; import java.util.ArrayList; -import java.util.Stack; /** * @author Simon Thoresen Hult @@ -47,7 +48,7 @@ public class ArithmeticNode implements ExpressionNode { @Override public Object evaluate(Context context) { StringBuilder ret = null; - Stack<ValueItem> buf = new Stack<>(); + Deque<ValueItem> buf = new ArrayDeque<>(); for (int i = 0; i < items.size(); ++i) { NodeItem item = items.get(i); Object val = item.node.evaluate(context); @@ -56,8 +57,7 @@ public class ArithmeticNode implements ExpressionNode { throw new IllegalArgumentException("Can not perform arithmetic on null value (referencing missing field?)"); } - if (val instanceof AttributeNode.VariableValueList) { - AttributeNode.VariableValueList value = (AttributeNode.VariableValueList)val; + if (val instanceof AttributeNode.VariableValueList value) { if (value.size() == 0) { throw new IllegalArgumentException("Can not perform arithmetic on missing field: " + item.node.toString()); @@ -102,27 +102,16 @@ public class ArithmeticNode implements ExpressionNode { return buf.pop().value; } - private void popOffTheTop(Stack<ValueItem> buf) { + private void popOffTheTop(Deque<ValueItem> buf) { ValueItem rhs = buf.pop(); ValueItem lhs = buf.pop(); switch (rhs.operator) { - case ADD: - lhs.value = lhs.value.doubleValue() + rhs.value.doubleValue(); - break; - case SUB: - lhs.value = lhs.value.doubleValue() - rhs.value.doubleValue(); - break; - case DIV: - lhs.value = lhs.value.doubleValue() / rhs.value.doubleValue(); - break; - case MUL: - lhs.value = lhs.value.doubleValue() * rhs.value.doubleValue(); - break; - case MOD: - lhs.value = lhs.value.longValue() % rhs.value.longValue(); - break; - default: - throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); + case ADD -> lhs.value = lhs.value.doubleValue() + rhs.value.doubleValue(); + case SUB -> lhs.value = lhs.value.doubleValue() - rhs.value.doubleValue(); + case DIV -> lhs.value = lhs.value.doubleValue() / rhs.value.doubleValue(); + case MUL -> lhs.value = lhs.value.doubleValue() * rhs.value.doubleValue(); + case MOD -> lhs.value = lhs.value.longValue() % rhs.value.longValue(); + default -> throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); } buf.push(lhs); } @@ -140,22 +129,15 @@ public class ArithmeticNode implements ExpressionNode { } public String operatorToString(int operator) { - switch (operator) { - case NOP: - return null; - case ADD: - return "+"; - case SUB: - return "-"; - case MOD: - return "%"; - case DIV: - return "/"; - case MUL: - return "*"; - default: - throw new IllegalStateException("Arithmetic operator " + operator + " not supported."); - } + return switch (operator) { + case NOP -> null; + case ADD -> "+"; + case SUB -> "-"; + case MOD -> "%"; + case DIV -> "/"; + case MUL -> "*"; + default -> throw new IllegalStateException("Arithmetic operator " + operator + " not supported."); + }; } private int stringToOperator(String operator) { @@ -192,8 +174,8 @@ public class ArithmeticNode implements ExpressionNode { } public static class NodeItem { - private int operator; - private ExpressionNode node; + private final int operator; + private final ExpressionNode node; NodeItem(int operator, ExpressionNode node) { this.operator = operator; diff --git a/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java index 5cfcef2a5e5..f97ebb20c28 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java @@ -7,9 +7,10 @@ import com.yahoo.document.select.Context; import com.yahoo.document.select.ResultList; import com.yahoo.document.select.Visitor; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.List; -import java.util.Stack; /** * This class defines a logical expression of nodes. This implementation uses a stack to evaluate its content as to @@ -56,7 +57,7 @@ public class LogicNode implements ExpressionNode { @Override public BucketSet getBucketSet(BucketIdFactory factory) { - Stack<BucketItem> buf = new Stack<>(); + Deque<BucketItem> buf = new ArrayDeque<>(); for (NodeItem item : items) { if (!buf.isEmpty()) { while (buf.peek().operator > item.operator) { @@ -76,11 +77,11 @@ public class LogicNode implements ExpressionNode { * * @param buf The stack of bucket items. */ - private void combineBuckets(Stack<BucketItem> buf) { + private void combineBuckets(Deque<BucketItem> buf) { BucketItem rhs = buf.pop(); BucketItem lhs = buf.pop(); switch (rhs.operator) { - case AND: + case AND -> { if (lhs.buckets == null) { lhs.buckets = rhs.buckets; } else if (rhs.buckets == null) { @@ -88,8 +89,8 @@ public class LogicNode implements ExpressionNode { } else { lhs.buckets = lhs.buckets.intersection(rhs.buckets); } - break; - case OR: + } + case OR -> { if (lhs.buckets == null) { // empty } else if (rhs.buckets == null) { @@ -97,16 +98,15 @@ public class LogicNode implements ExpressionNode { } else { lhs.buckets = lhs.buckets.union(rhs.buckets); } - break; - default: - throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); + } + default -> throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported."); } buf.push(lhs); } @Override public Object evaluate(Context context) { - Stack<ValueItem> buf = new Stack<>(); + Deque<ValueItem> buf = new ArrayDeque<>(); for (NodeItem item : items) { if ( buf.size() > 1) { while ((buf.peek().getOperator() >= item.operator)) { @@ -126,7 +126,7 @@ public class LogicNode implements ExpressionNode { * * @param buf The stack of values. */ - private void combineValues(Stack<ValueItem> buf) { + private void combineValues(Deque<ValueItem> buf) { ValueItem rhs = buf.pop(); ValueItem lhs = buf.pop(); buf.push(new LazyCombinedItem(lhs, rhs)); @@ -156,16 +156,12 @@ public class LogicNode implements ExpressionNode { * @return The string representation. */ private String operatorToString(int operator) { - switch (operator) { - case NOP: - return null; - case OR: - return "or"; - case AND: - return "and"; - default: - throw new IllegalStateException("Logical operator " + operator + " not supported."); - } + return switch (operator) { + case NOP -> null; + case OR -> "or"; + case AND -> "and"; + default -> throw new IllegalStateException("Logical operator " + operator + " not supported."); + }; } /** @@ -231,14 +227,10 @@ public class LogicNode implements ExpressionNode { public ResultList getResult() { if (lazyResult == null) { switch (rhs.getOperator()) { - case AND: - lazyResult = lhs.getResult().combineAND(rhs); - break; - case OR: - lazyResult = lhs.getResult().combineOR(rhs); - break; - default: - throw new IllegalStateException("Logical operator " + rhs.getOperator() + " not supported."); + case AND -> lazyResult = lhs.getResult().combineAND(rhs); + case OR -> lazyResult = lhs.getResult().combineOR(rhs); + default -> + throw new IllegalStateException("Logical operator " + rhs.getOperator() + " not supported."); } } return lazyResult; @@ -249,7 +241,7 @@ public class LogicNode implements ExpressionNode { * Private class to store bucket sets in a stack. */ private static final class BucketItem { - private int operator; + final private int operator; private BucketSet buckets; BucketItem(int operator, BucketSet buckets) { @@ -262,8 +254,8 @@ public class LogicNode implements ExpressionNode { * Private class to store expression nodes in a stack. */ public static final class NodeItem { - private int operator; - private ExpressionNode node; + final private int operator; + final private ExpressionNode node; NodeItem(int operator, ExpressionNode node) { this.operator = operator; diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index 829cf9d306c..bbe4f5373cb 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -282,9 +282,9 @@ VespaDocumentDeserializer::read(StringFieldValue &value) { setValue(value, val, _stream.isLongLivedBuffer()); if (coding & 0x40) { uint32_t serializedAnnotationsSize = readValue<uint32_t>(_stream); - value.setSpanTrees(vespalib::ConstBufferRef(_stream.peek(), serializedAnnotationsSize), - _repo, _version, _stream.isLongLivedBuffer()); - _stream.adjustReadPos(serializedAnnotationsSize); + auto span_buf = vespalib::ConstBufferRef(_stream.peek(), serializedAnnotationsSize); + _stream.adjustReadPos(serializedAnnotationsSize); // Trigger any out-of-bounds before using buffer range. + value.setSpanTrees(span_buf, _repo, _version, _stream.isLongLivedBuffer()); } else { value.clearSpanTrees(); } @@ -359,11 +359,13 @@ VespaDocumentDeserializer::readStructNoReset(StructFieldValue &value) { if (is_compressed && (compression_type != CompressionConfig::LZ4)) [[unlikely]] { throw DeserializeException(fmt("Unsupported compression type: %u", static_cast<uint8_t>(compression_type)), VESPA_STRLOC); } + // Must read field info _prior_ to checking remaining stream size against + // data_size, as the field info size is not counted as part of data_size. + readFieldInfo(_stream, field_info, is_compressed ? uncompressed_size : data_size); if (data_size > _stream.size()) [[unlikely]] { throw DeserializeException(fmt("Struct size (%zu) is greater than remaining buffer size (%zu)", data_size, _stream.size()), VESPA_STRLOC); } - readFieldInfo(_stream, field_info, is_compressed ? uncompressed_size : data_size); if (data_size > 0) { ByteBuffer buffer = is_compressed ? deCompress(compression_type, uncompressed_size, ConstBufferRef(_stream.peek(), data_size)) diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java index 03cd9b48de2..a50e3cd6c69 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java @@ -6,11 +6,12 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.Vector; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -287,7 +288,7 @@ public class VisitorIteratorTestCase { // keeping some in the active set and some in pending int pendingTotal = buckets / 8; int activeTotal = buckets / 8; - Vector<VisitorIterator.BucketProgress> trackedBuckets = new Vector<VisitorIterator.BucketProgress>(); + List<VisitorIterator.BucketProgress> trackedBuckets = new ArrayList<>(); // Pre-fetch, since otherwise we'd reuse pending buckets for (int i = 0; i < pendingTotal + activeTotal; ++i) { @@ -760,8 +761,8 @@ public class VisitorIteratorTestCase { assertTrue(bk4.compareTo(bk3) > 0); ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL); ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L); - assertTrue(bk5.compareTo(bk2) == 0); - assertTrue(bk6.compareTo(bk3) == 0); + assertEquals(0, bk5.compareTo(bk2)); + assertEquals(0, bk6.compareTo(bk3)); } private void doTestBucketKeyGeneration(int db) { @@ -1384,16 +1385,17 @@ public class VisitorIteratorTestCase { @Test public void testImportProgressWithOutdatedDistribution() throws ParseException { - String input = "VDS bucket progress file\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0\n" + - "28000000000001be:0\n"; - - int db = 12; + String input = """ + VDS bucket progress file + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0 + 28000000000001be:0 + """; + BucketIdFactory idFactory = new BucketIdFactory(); ProgressToken p = new ProgressToken(input); assertEquals(10, p.getDistributionBitCount()); @@ -1426,14 +1428,15 @@ public class VisitorIteratorTestCase { public void testImportInconsistentProgressIncrease() throws ParseException { // Bucket progress "file" that upon time of changing from 4 to 7 // distribution bits and writing the progress had an active bucket - String input = "VDS bucket progress file\n" + - "7\n" + - "32\n" + - "24\n" + - "128\n" + - "100000000000000c:0\n"; + String input = """ + VDS bucket progress file + 7 + 32 + 24 + 128 + 100000000000000c:0 + """; // Now we're at 8 distribution bits - int db = 8; BucketIdFactory idFactory = new BucketIdFactory(); ProgressToken p = new ProgressToken(input); assertEquals(7, p.getDistributionBitCount()); @@ -1472,12 +1475,14 @@ public class VisitorIteratorTestCase { public void testImportInconsistentProgressDecrease() throws ParseException { // Bucket progress "file" that upon time of changing from 4 to 7 // distribution bits and writing the progress had an active bucket - String input = "VDS bucket progress file\n" + - "7\n" + - "32\n" + - "24\n" + - "128\n" + - "100000000000000c:0\n"; + String input = """ + VDS bucket progress file + 7 + 32 + 24 + 128 + 100000000000000c:0 + """; BucketIdFactory idFactory = new BucketIdFactory(); ProgressToken p = new ProgressToken(input); @@ -1553,8 +1558,15 @@ public class VisitorIteratorTestCase { // Try to pass a known document selection to an unknown docsel iterator boolean caughtIt = false; try { - ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n" - + "8000000000001f49:0\n8000000000001a85:0\n"); + ProgressToken p = new ProgressToken(""" + VDS bucket progress file + 16 + 3 + 1 + 3 + 8000000000001f49:0 + 8000000000001a85:0 + """); VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p); } @@ -1566,14 +1578,16 @@ public class VisitorIteratorTestCase { // Now try it the other way around caughtIt = false; try { - ProgressToken p = new ProgressToken("VDS bucket progress file\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0\n" + - "28000000000001be:0\n"); + ProgressToken p = new ProgressToken(""" + VDS bucket progress file + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0 + 28000000000001be:0 + """); VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p); } @@ -1645,13 +1659,14 @@ public class VisitorIteratorTestCase { public void testMalformedProgressFile() { boolean caughtIt = false; try { - new ProgressToken("VDS bucket progress file\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:"); + new ProgressToken(""" + VDS bucket progress file + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:"""); } catch (IllegalArgumentException e) { caughtIt = true; } @@ -1662,9 +1677,11 @@ public class VisitorIteratorTestCase { public void testFailOnTooFewLinesInFile() { boolean caughtIt = false; try { - new ProgressToken("VDS bucket progress file\n" + - "10\n" + - "503\n"); + new ProgressToken(""" + VDS bucket progress file + 10 + 503 + """); } catch (IllegalArgumentException e) { caughtIt = true; } @@ -1675,13 +1692,14 @@ public class VisitorIteratorTestCase { public void testUnknownFirstHeaderLine() { boolean caughtIt = false; try { - new ProgressToken("Smurf Time 3000\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0"); + new ProgressToken(""" + Smurf Time 3000 + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0"""); } catch (IllegalArgumentException e) { caughtIt = true; } @@ -1690,14 +1708,16 @@ public class VisitorIteratorTestCase { @Test public void testBinaryProgressSerialization() { - String input = "VDS bucket progress file (48.828125% completed)\n" + - "10\n" + - "503\n" + - "500\n" + - "1024\n" + - "28000000000000be:0\n" + - "28000000000002be:0\n" + - "28000000000001be:0\n"; + String input = """ + VDS bucket progress file (48.828125% completed) + 10 + 503 + 500 + 1024 + 28000000000000be:0 + 28000000000002be:0 + 28000000000001be:0 + """; ProgressToken p = new ProgressToken(input); byte[] buf = p.serialize(); ProgressToken p2 = new ProgressToken(buf); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 1b6593581ff..dbf3f0486ab 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -288,16 +288,16 @@ public class Flags { APPLICATION_ID); public static final UnboundListFlag<String> FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES = defineListFlag( - "file-distribution-accepted-compression-types", List.of("gzip", "lz4"), String.class, - List.of("hmusum"), "2022-07-05", "2023-02-01", - "´List of accepted compression types used when asking for a file reference. Valid values: gzip, lz4", + "file-distribution-accepted-compression-types", List.of("gzip", "lz4", "zstd"), String.class, + List.of("hmusum"), "2022-07-05", "2023-02-15", + "´List of accepted compression types used when asking for a file reference. Valid values: gzip, lz4, zstd", "Takes effect on restart of service", APPLICATION_ID); public static final UnboundListFlag<String> FILE_DISTRIBUTION_COMPRESSION_TYPES_TO_SERVE = defineListFlag( - "file-distribution-compression-types-to-use", List.of("lz4", "gzip"), String.class, - List.of("hmusum"), "2022-07-05", "2023-02-01", - "List of compression types to use (in preferred order), matched with accepted compression types when serving file references. Valid values: gzip, lz4", + "file-distribution-compression-types-to-use", List.of("zstd", "lz4", "gzip"), String.class, + List.of("hmusum"), "2022-07-05", "2023-02-15", + "List of compression types to use (in preferred order), matched with accepted compression types when serving file references. Valid values: gzip, lz4, zstd", "Takes effect on restart of service", APPLICATION_ID); diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 6589e5bf1a3..bce3a7dc703 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -36,6 +36,7 @@ <target_jdk_version>17</target_jdk_version> <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> + <maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version> <!-- NOTE: this must not be overriden by users, and must be in sync with junit version specified in 'tenant-cd-api' --> <vespa.junit.version>5.8.1</vespa.junit.version> <test.categories>!integration</test.categories> @@ -443,6 +444,15 @@ </profiles> <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>${maven-dependency-plugin.version}</version> + </plugin> + </plugins> + </pluginManagement> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java index aab4bf710c1..c62be40f1db 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/AbstractHttpClient.java @@ -114,6 +114,7 @@ public abstract class AbstractHttpClient implements HttpClient { throw new IllegalStateException("No hosts to perform the request against"); } + @SuppressWarnings("deprecation") private HttpClientContext contextWithTimeout(RequestBuilder builder) { HttpClientContext context = HttpClientContext.create(); RequestConfig config = builder.config; @@ -121,6 +122,7 @@ public abstract class AbstractHttpClient implements HttpClient { Optional<Duration> remaining = builder.deadline.timeLeftOrThrow(); if (remaining.isPresent()) { config = RequestConfig.copy(config) + .setConnectTimeout(min(config.getConnectTimeout(), remaining.get())) .setConnectionRequestTimeout(min(config.getConnectionRequestTimeout(), remaining.get())) .setResponseTimeout(min(config.getResponseTimeout(), remaining.get())) .build(); diff --git a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java index 2d1e80101cf..1f36ae8f8a4 100644 --- a/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java +++ b/http-client/src/main/java/ai/vespa/hosted/client/HttpClient.java @@ -34,10 +34,12 @@ import static java.util.Objects.requireNonNull; /** * @author jonmv */ +@SuppressWarnings("deprecation") public interface HttpClient extends Closeable { RequestConfig defaultRequestConfig = RequestConfig.custom() .setConnectionRequestTimeout(Timeout.ofSeconds(5)) + .setConnectTimeout(Timeout.ofSeconds(5)) .setRedirectsEnabled(false) .build(); diff --git a/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java b/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java index 8275098cb8d..0c140ce236e 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java +++ b/http-utils/src/main/java/ai/vespa/util/http/hc5/HttpToHttpsRoutePlanner.java @@ -2,6 +2,7 @@ package ai.vespa.util.http.hc5; import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.client5.http.impl.DefaultSchemePortResolver; import org.apache.hc.client5.http.protocol.HttpClientContext; import org.apache.hc.client5.http.routing.HttpRoutePlanner; import org.apache.hc.core5.http.HttpHost; @@ -15,19 +16,17 @@ import org.apache.hc.core5.http.protocol.HttpContext; */ class HttpToHttpsRoutePlanner implements HttpRoutePlanner { - @SuppressWarnings("deprecation") @Override + @SuppressWarnings("deprecation") public HttpRoute determineRoute(HttpHost target, HttpContext context) { if ( ! target.getSchemeName().equals("http") && ! target.getSchemeName().equals("https")) throw new IllegalArgumentException("Scheme must be 'http' or 'https' when using HttpToHttpsRoutePlanner, was '" + target.getSchemeName() + "'"); - if (target.getPort() == -1) - throw new IllegalArgumentException("Port must be set when using HttpToHttpsRoutePlanner"); - if (HttpClientContext.adapt(context).getRequestConfig().getProxy() != null) throw new IllegalArgumentException("Proxies are not supported with HttpToHttpsRoutePlanner"); - return new HttpRoute(new HttpHost("https", target.getAddress(), target.getHostName(), target.getPort())); + int port = DefaultSchemePortResolver.INSTANCE.resolve(target); + return new HttpRoute(new HttpHost("https", target.getAddress(), target.getHostName(), port)); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index 2e4bb701454..328cd00742f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.TensorDataType; +import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.language.process.Embedder; @@ -13,6 +14,7 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -33,7 +35,7 @@ public class EmbedExpression extends Expression { private TensorType targetType; public EmbedExpression(Map<String, Embedder> embedders, String embedderId) { - super(DataType.STRING); + super(null); this.embedderId = embedderId; boolean embedderIdProvided = embedderId != null && embedderId.length() > 0; @@ -43,14 +45,14 @@ public class EmbedExpression extends Expression { } else if (embedders.size() > 1 && ! embedderIdProvided) { this.embedder = new Embedder.FailingEmbedder("Multiple embedders are provided but no embedder id is given. " + - "Valid embedders are " + validEmbedders(embedders)); + "Valid embedders are " + validEmbedders(embedders)); } else if (embedders.size() == 1 && ! embedderIdProvided) { this.embedder = embedders.entrySet().stream().findFirst().get().getValue(); } else if ( ! embedders.containsKey(embedderId)) { this.embedder = new Embedder.FailingEmbedder("Can't find embedder '" + embedderId + "'. " + - "Valid embedders are " + validEmbedders(embedders)); + "Valid embedders are " + validEmbedders(embedders)); } else { this.embedder = embedders.get(embedderId); } @@ -64,11 +66,48 @@ public class EmbedExpression extends Expression { @Override protected void doExecute(ExecutionContext context) { - StringFieldValue input = (StringFieldValue) context.getValue(); - Tensor tensor = embedder.embed(input.getString(), - new Embedder.Context(destination).setLanguage(context.getLanguage()), - targetType); - context.setValue(new TensorFieldValue(tensor)); + Tensor output; + if (context.getValue().getDataType() == DataType.STRING) { + output = embedSingleValue(context); + } + else if (context.getValue().getDataType() instanceof ArrayDataType && + ((ArrayDataType)context.getValue().getDataType()).getNestedType() == DataType.STRING) { + output = embedArrayValue(context); + } + else { + throw new IllegalArgumentException("Embedding can only be done on string or string array fields, not " + + context.getValue().getDataType()); + } + context.setValue(new TensorFieldValue(output)); + } + + private Tensor embedSingleValue(ExecutionContext context) { + StringFieldValue input = (StringFieldValue)context.getValue(); + return embed(input.getString(), targetType, context); + } + + @SuppressWarnings("unchecked") + private Tensor embedArrayValue(ExecutionContext context) { + var input = (Array<StringFieldValue>)context.getValue(); + var builder = Tensor.Builder.of(targetType); + for (int i = 0; i < input.size(); i++) { + Tensor tensor = embed(input.get(i).getString(), targetType.indexedSubtype(), context); + for (Iterator<Tensor.Cell> cells = tensor.cellIterator(); cells.hasNext(); ) { + Tensor.Cell cell = cells.next(); + builder.cell() + .label(targetType.mappedSubtype().dimensions().get(0).name(), i) + .label(targetType.indexedSubtype().dimensions().get(0).name(), cell.getKey().label(0)) + .value(cell.getValue()); + } + } + return builder.build(); + } + + private Tensor embed(String input, TensorType targetType, ExecutionContext context) { + return embedder.embed(input, + new Embedder.Context(destination).setLanguage(context.getLanguage()), + targetType); + } @Override @@ -78,6 +117,9 @@ public class EmbedExpression extends Expression { throw new VerificationException(this, "No output field in this statement: " + "Don't know what tensor type to embed into."); targetType = toTargetTensor(context.getInputType(this, outputField)); + if ( ! validTarget(targetType)) + throw new VerificationException(this, "The embedding target field must either be a dense 1d tensor, " + + "an array of dense 1d tensors, or a mixed 2d tensor"); context.setValueType(createdOutputType()); } @@ -94,6 +136,14 @@ public class EmbedExpression extends Expression { } + private boolean validTarget(TensorType target) { + if (target.dimensions().size() == 1 && target.indexedSubtype().rank() == 1) + return true; + if (target.dimensions().size() == 2 && target.indexedSubtype().rank() == 1 && target.mappedSubtype().rank() == 1) + return true; + return false; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -105,7 +155,7 @@ public class EmbedExpression extends Expression { } @Override - public int hashCode() { return 1; } + public int hashCode() { return 98857339; } @Override public boolean equals(Object o) { return o instanceof EmbedExpression; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java index 47d4e789602..ded665b2d77 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage.expressions; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.Stack; /** * @author Simon Thoresen Hult @@ -20,7 +21,7 @@ public class MathResolver { } public Expression resolve() { - Stack<Item> stack = new Stack<>(); + Deque<Item> stack = new ArrayDeque<>(); stack.push(items.remove(0)); while (!items.isEmpty()) { Item item = items.remove(0); @@ -32,10 +33,10 @@ public class MathResolver { while (stack.size() > 1) { pop(stack); } - return stack.remove(0).exp; + return stack.pop().exp; } - private void pop(Stack<Item> stack) { + private void pop(Deque<Item> stack) { Item rhs = stack.pop(); Item lhs = stack.peek(); lhs.exp = new ArithmeticExpression(lhs.exp, rhs.op, rhs.exp); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index 3a32b0049fe..c446c04065a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -68,7 +68,7 @@ public class ScriptTestCase { exp.verify(input); fail(); } catch (VerificationException e) { - assertTrue(e.getExpressionType().equals(ScriptExpression.class)); + assertEquals(e.getExpressionType(), ScriptExpression.class); assertEquals("Expected any input, got null.", e.getMessage()); } } @@ -268,6 +268,40 @@ public class ScriptTestCase { assertEquals(Tensor.from(tensorType, "[115, 101, 99, 111]"), tensorArray.get(1).getTensor().get()); } + @Test + public void testArrayEmbedToSparseTensor() throws ParseException { + Map<String, Embedder> embedders = Map.of("emb1", new MockEmbedder("myDocument.mySparseTensor")); + + TensorType tensorType = TensorType.fromSpec("tensor(passage{}, d[4])"); + var expression = Expression.fromString("input myTextArray | embed | attribute 'mySparseTensor'", + new SimpleLinguistics(), + embedders); + + SimpleTestAdapter adapter = new SimpleTestAdapter(); + adapter.createField(new Field("myTextArray", new ArrayDataType(DataType.STRING))); + + var tensorField = new Field("mySparseTensor", new TensorDataType(tensorType)); + adapter.createField(tensorField); + + var array = new Array<StringFieldValue>(new ArrayDataType(DataType.STRING)); + array.add(new StringFieldValue("first")); + array.add(new StringFieldValue("second")); + adapter.setValue("myTextArray", array); + expression.setStatementOutput(new DocumentType("myDocument"), tensorField); + + // Necessary to resolve output type + VerificationContext verificationContext = new VerificationContext(adapter); + assertEquals(new TensorDataType(tensorType), expression.verify(verificationContext)); + + ExecutionContext context = new ExecutionContext(adapter); + context.setValue(array); + expression.execute(context); + assertTrue(adapter.values.containsKey("mySparseTensor")); + var sparseTensor = (TensorFieldValue)adapter.values.get("mySparseTensor"); + assertEquals(Tensor.from(tensorType, "{ '0':[102, 105, 114, 115], '1':[115, 101, 99, 111]}"), + sparseTensor.getTensor().get()); + } + // An embedder which returns the char value of each letter in the input. */ private static class MockEmbedder implements Embedder { diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 5d749084eaf..5ae3e056309 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -37,10 +37,6 @@ <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> </exclusions> </dependency> <dependency> @@ -63,11 +59,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <scope>provided</scope> - </dependency> </dependencies> <build> diff --git a/jrt/tests/com/yahoo/jrt/ConnectTest.java b/jrt/tests/com/yahoo/jrt/ConnectTest.java index 6ad7198f310..034cab29956 100644 --- a/jrt/tests/com/yahoo/jrt/ConnectTest.java +++ b/jrt/tests/com/yahoo/jrt/ConnectTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ConnectTest { @@ -25,12 +24,12 @@ public class ConnectTest { target.close(); for (int i = 0; i < 100; i++) { - if (!target.isClosed()) { + if (target.isClosed()) { break; } try { Thread.sleep(100); } catch (InterruptedException e) {} } - assertFalse(target.isClosed()); + assertTrue(target.isClosed()); acceptor.shutdown().join(); client.transport().shutdown().join(); diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index bc93ff58719..c6f41406f4f 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -18,16 +18,16 @@ org.apache.commons:commons-collections4:4.2 org.apache.commons:commons-compress:1.22 org.apache.commons:commons-lang3:3.12.0 org.apache.maven:maven-archiver:3.6.0 -org.apache.maven:maven-artifact:3.8.6 -org.apache.maven:maven-builder-support:3.8.6 -org.apache.maven:maven-core:3.8.6 -org.apache.maven:maven-model:3.8.6 -org.apache.maven:maven-model-builder:3.8.6 -org.apache.maven:maven-plugin-api:3.8.6 -org.apache.maven:maven-repository-metadata:3.8.6 -org.apache.maven:maven-resolver-provider:3.8.6 -org.apache.maven:maven-settings:3.8.6 -org.apache.maven:maven-settings-builder:3.8.6 +org.apache.maven:maven-artifact:3.8.7 +org.apache.maven:maven-builder-support:3.8.7 +org.apache.maven:maven-core:3.8.7 +org.apache.maven:maven-model:3.8.7 +org.apache.maven:maven-model-builder:3.8.7 +org.apache.maven:maven-plugin-api:3.8.7 +org.apache.maven:maven-repository-metadata:3.8.7 +org.apache.maven:maven-resolver-provider:3.8.7 +org.apache.maven:maven-settings:3.8.7 +org.apache.maven:maven-settings-builder:3.8.7 org.apache.maven.enforcer:enforcer-api:3.0.0 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 org.apache.maven.plugins:maven-shade-plugin:3.4.1 diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 1f63744c205..ac8fd637646 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -21,7 +21,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; import java.util.List; -import java.util.Stack; import java.util.concurrent.atomic.AtomicInteger; /** @@ -130,7 +129,7 @@ public class RoutingNode implements ReplyHandler { * @param msg The error message to assign. */ private void notifyAbort(String msg) { - Stack<RoutingNode> stack = new Stack<>(); + Deque<RoutingNode> stack = new ArrayDeque<>(); stack.push(this); while (!stack.isEmpty()) { RoutingNode node = stack.pop(); @@ -430,8 +429,7 @@ public class RoutingNode implements ReplyHandler { private boolean lookupRoute() { RoutingTable table = mbus.getRoutingTable(msg.getProtocol()); Hop hop = route.getHop(0); - if (hop.getDirective(0) instanceof RouteDirective) { - RouteDirective dir = (RouteDirective)hop.getDirective(0); + if (hop.getDirective(0) instanceof RouteDirective dir) { if (table == null || !table.hasRoute(dir.getName())) { setError(ErrorCode.ILLEGAL_ROUTE, "Route '" + dir.getName() + "' does not exist."); return false; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 9877dd69e83..924eed18633 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -182,7 +182,7 @@ public class RankProfilesConfigImporter { options.setExecutionMode(onnxModelConfig.stateless_execution_mode()); options.setInterOpThreads(onnxModelConfig.stateless_interop_threads()); options.setIntraOpThreads(onnxModelConfig.stateless_intraop_threads()); - options.setGpuDevice(onnxModelConfig.gpu_device(), onnxModelConfig.gpu_device_required()); + options.setGpuDevice(onnxModelConfig.gpu_device()); return new OnnxModel(name, file, options); } catch (InterruptedException e) { throw new IllegalStateException("Gave up waiting for ONNX model " + onnxModelConfig.name()); diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 125707c9aaa..ebed464421b 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -30,18 +30,8 @@ public class OnnxEvaluator { } public OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options) { - try { - if (options == null) { - options = new OnnxEvaluatorOptions(); - } - environment = OrtEnvironment.getEnvironment(); - session = environment.createSession(modelPath, options.getOptions()); - } catch (OrtException e) { - if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) { - throw new IllegalArgumentException("No such file: "+modelPath); - } - throw new RuntimeException("ONNX Runtime exception", e); - } + environment = OrtEnvironment.getEnvironment(); + session = createSession(modelPath, environment, options); } public Tensor evaluate(Map<String, Tensor> inputs, String output) { @@ -96,6 +86,31 @@ public class OnnxEvaluator { } } + private static OrtSession createSession(String modelPath, OrtEnvironment environment, OnnxEvaluatorOptions options) { + if (options == null) { + options = new OnnxEvaluatorOptions(); + } + try { + return environment.createSession(modelPath, options.getOptions()); + } catch (OrtException e) { + if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) { + throw new IllegalArgumentException("No such file: " + modelPath); + } + if (isCudaError(e)) { + throw new IllegalArgumentException("GPU device " + options.gpuDevice() + " requested, but CUDA initialization failed", e); + } + throw new RuntimeException("ONNX Runtime exception", e); + } + } + + private static boolean isCudaError(OrtException e) { + return switch (e.getCode()) { + case ORT_FAIL -> e.getMessage().contains("cudaError"); + case ORT_EP_FAIL -> e.getMessage().contains("Failed to find CUDA"); + default -> false; + }; + } + public static boolean isRuntimeAvailable() { return isRuntimeAvailable(""); } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java index cdbce760d92..f838a3b3f7f 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java @@ -16,8 +16,7 @@ public class OnnxEvaluatorOptions { private OrtSession.SessionOptions.ExecutionMode executionMode; private int interOpThreads; private int intraOpThreads; - private int gpuDeviceNumber; - private boolean gpuDeviceRequired; + private int gpuDevice; public OnnxEvaluatorOptions() { // Defaults: @@ -25,8 +24,7 @@ public class OnnxEvaluatorOptions { executionMode = OrtSession.SessionOptions.ExecutionMode.SEQUENTIAL; interOpThreads = 1; intraOpThreads = Math.max(1, (int) Math.ceil(((double) Runtime.getRuntime().availableProcessors()) / 4)); - gpuDeviceNumber = -1; - gpuDeviceRequired = false; + gpuDevice = -1; } public OrtSession.SessionOptions getOptions() throws OrtException { @@ -35,19 +33,10 @@ public class OnnxEvaluatorOptions { options.setExecutionMode(executionMode); options.setInterOpNumThreads(interOpThreads); options.setIntraOpNumThreads(intraOpThreads); - addCuda(options); - return options; - } - - private void addCuda(OrtSession.SessionOptions options) { - if (gpuDeviceNumber < 0) return; - try { - options.addCUDA(gpuDeviceNumber); - } catch (OrtException e) { - if (gpuDeviceRequired) { - throw new IllegalArgumentException("GPU device " + gpuDeviceNumber + " is required, but CUDA backend could not be initialized", e); - } + if (gpuDevice > -1) { + options.addCUDA(gpuDevice); } + return options; } public void setExecutionMode(String mode) { @@ -70,9 +59,12 @@ public class OnnxEvaluatorOptions { } } - public void setGpuDevice(int deviceNumber, boolean required) { - this.gpuDeviceNumber = deviceNumber; - this.gpuDeviceRequired = required; + public void setGpuDevice(int deviceNumber) { + this.gpuDevice = deviceNumber; + } + + public int gpuDevice() { + return gpuDevice; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/WireguardMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java index f7f1a421cd8..83d3a98d85d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/WireguardMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/ContainerWireguardTask.java @@ -7,7 +7,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; * * @author gjoranv */ -public interface WireguardMaintainer { +public interface ContainerWireguardTask { void converge(NodeAgentContext context); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 4e00290c86e..ec1e42a911e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -24,7 +24,7 @@ import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.WireguardMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.ContainerWireguardTask; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; @@ -72,7 +72,7 @@ public class NodeAgentImpl implements NodeAgent { private final Duration warmUpDuration; private final DoubleFlag containerCpuCap; private final VespaServiceDumper serviceDumper; - private final Optional<WireguardMaintainer> wireguardMaintainer; + private final Optional<ContainerWireguardTask> wireguardMaintainer; private Thread loopThread; private ContainerState containerState = UNKNOWN; @@ -108,7 +108,7 @@ public class NodeAgentImpl implements NodeAgent { RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer, FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers, Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, - VespaServiceDumper serviceDumper, Optional<WireguardMaintainer> wireguardMaintainer) { + VespaServiceDumper serviceDumper, Optional<ContainerWireguardTask> wireguardMaintainer) { this(contextSupplier, nodeRepository, orchestrator, containerOperations, registryCredentialsProvider, storageMaintainer, flagSource, credentialsMaintainers, aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION, serviceDumper, wireguardMaintainer); @@ -120,7 +120,7 @@ public class NodeAgentImpl implements NodeAgent { FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers, Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, Duration warmUpDuration, VespaServiceDumper serviceDumper, - Optional<WireguardMaintainer> wireguardMaintainer) { + Optional<ContainerWireguardTask> wireguardMaintainer) { this.contextSupplier = contextSupplier; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index ab702c85ffc..75da8d2c068 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -14,18 +14,18 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.Stack; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -167,7 +167,7 @@ public class FileFinder { // Only need to traverse as deep as we want to match, unless we want to match everything in directories // already matched Files.walkFileTree(basePath, Set.of(), maxDepth, new SimpleFileVisitor<>() { - private final Stack<FileAttributes> matchingDirectoryStack = new Stack<>(); + private final Deque<FileAttributes> matchingDirectoryStack = new ArrayDeque<>(); private int currentLevel = -1; @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java index 1546da7d9a8..e70a05dcb7c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/RetryingClusterControllerClientFactory.java @@ -20,7 +20,7 @@ import java.util.logging.Logger; */ public class RetryingClusterControllerClientFactory extends AbstractComponent implements ClusterControllerClientFactory { - private static Logger log = Logger.getLogger(RetryingClusterControllerClientFactory.class.getName()); + private static final Logger log = Logger.getLogger(RetryingClusterControllerClientFactory.class.getName()); private final HttpClient client; diff --git a/parent/pom.xml b/parent/pom.xml index c9b406a7ace..93945575588 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1098,8 +1098,8 @@ <properties> <airline.version>0.9</airline.version> - <antlr.version>3.5.2</antlr.version> - <antlr4.version>4.9.3</antlr4.version> + <antlr.version>3.5.3</antlr.version> + <antlr4.version>4.11.1</antlr4.version> <apache.httpclient.version>4.5.14</apache.httpclient.version> <apache.httpcore.version>4.4.16</apache.httpcore.version> <apache.httpclient5.version>5.2.1</apache.httpclient5.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> @@ -1136,15 +1136,15 @@ <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> <maven-bundle-plugin.version>5.1.2</maven-bundle-plugin.version> <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> - <maven-core.version>3.8.6</maven-core.version> - <maven-dependency-plugin.version>3.3.0</maven-dependency-plugin.version> <!-- NOTE: When upgrading, also update explicit versions in tenant base poms! --> + <maven-core.version>3.8.7</maven-core.version> + <maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version> <!-- NOTE: When upgrading, also update explicit versions in tenant base poms! --> <maven-deploy-plugin.version>2.8.2</maven-deploy-plugin.version> <maven-enforcer-plugin.version>3.0.0</maven-enforcer-plugin.version> <maven-failsafe-plugin.version>3.0.0-M6</maven-failsafe-plugin.version> <maven-install-plugin.version>3.0.0-M1</maven-install-plugin.version> <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version> <maven-javadoc-plugin.version>3.3.1</maven-javadoc-plugin.version> - <maven-plugin-api.version>3.8.6</maven-plugin-api.version> + <maven-plugin-api.version>${maven-core.version}</maven-plugin-api.version> <maven-plugin-tools.version>3.6.4</maven-plugin-tools.version> <maven-resources-plugin.version>3.2.0</maven-resources-plugin.version> <maven-shade-plugin.version>3.4.1</maven-shade-plugin.version> @@ -119,7 +119,6 @@ <module>standalone-container</module> <module>storage</module> <module>streamingvisitors</module> - <module>tenant-base</module> <module>tenant-cd-api</module> <module>tenant-cd-commons</module> <module>testutil</module> diff --git a/searchcore/src/vespa/searchcore/config/onnx-models.def b/searchcore/src/vespa/searchcore/config/onnx-models.def index b8f5d319075..85b061fcd7c 100644 --- a/searchcore/src/vespa/searchcore/config/onnx-models.def +++ b/searchcore/src/vespa/searchcore/config/onnx-models.def @@ -12,4 +12,3 @@ model[].stateless_execution_mode string default="" model[].stateless_interop_threads int default=-1 model[].stateless_intraop_threads int default=-1 model[].gpu_device int default=-1 -model[].gpu_device_required bool default=false diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 398404c6e71..596525e17d7 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -240,14 +240,18 @@ DocsumStoreVsmDocument::insert_juniper_field(const vespalib::string& field_name, auto field_value = get_field_value(field_name); if (field_value) { FieldModifier* modifier = nullptr; - if (is_struct_or_multivalue_field_type(*field_value->getDataType())) { - auto entry_idx = _result_class.getIndexFromName(field_name.c_str()); - if (entry_idx >= 0) { - assert((uint32_t) entry_idx < _result_class.getNumEntries()); + auto entry_idx = _result_class.getIndexFromName(field_name.c_str()); + if (entry_idx >= 0) { + assert((uint32_t) entry_idx < _result_class.getNumEntries()); + if (is_struct_or_multivalue_field_type(*field_value->getDataType())) { modifier = _docsum_filter.get_field_modifier(entry_idx); + } else { + if (!_docsum_filter.has_flatten_juniper_command(entry_idx)) { + modifier = _docsum_filter.get_field_modifier(entry_idx); + } else { + // Markup for juniper has already been added due to FLATTENJUNIPER command in vsm summary config. + } } - } else { - // Markup for juniper has already been added due to FLATTENJUNIPER command in vsm summary config. } SnippetModifierJuniperConverter string_converter(converter, modifier); SlimeFiller::insert_juniper_field(*field_value, inserter, string_converter); @@ -407,6 +411,14 @@ DocsumFilter::insert_summary_field(uint32_t entry_idx, const Document& doc, vesp _flattenWriter.clear(); } +bool +DocsumFilter::has_flatten_juniper_command(uint32_t entry_idx) const +{ + const auto& field_spec = _fields[entry_idx]; + auto command = field_spec.getCommand(); + return command == VsmsummaryConfig::Fieldmap::Command::FLATTENJUNIPER; +} + FieldModifier* DocsumFilter::get_field_modifier(uint32_t entry_idx) { diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h index e87a3e9a431..584f7c8141e 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h @@ -67,6 +67,7 @@ public: search::docsummary::DocsumStoreFieldValue get_summary_field(uint32_t entry_idx, const Document& doc); void insert_summary_field(uint32_t entry_idx, const Document& doc, vespalib::slime::Inserter& inserter); + bool has_flatten_juniper_command(uint32_t entry_idx) const; FieldModifier* get_field_modifier(uint32_t entry_idx); }; diff --git a/tenant-base/OWNERS b/tenant-base/OWNERS deleted file mode 100644 index d0a102ecbf4..00000000000 --- a/tenant-base/OWNERS +++ /dev/null @@ -1 +0,0 @@ -jonmv diff --git a/tenant-base/README b/tenant-base/README deleted file mode 100644 index fcf50a7d00a..00000000000 --- a/tenant-base/README +++ /dev/null @@ -1 +0,0 @@ -Parent pom for hosted Vespa applications diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml deleted file mode 100644 index cf7898f6951..00000000000 --- a/tenant-base/pom.xml +++ /dev/null @@ -1,442 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-base</artifactId> - <version>8-SNAPSHOT</version> - <name>Hosted Vespa tenant base</name> - <description>Parent POM for all hosted Vespa applications.</description> - <url>https://github.com/vespa-engine</url> - <packaging>pom</packaging> - - <licenses> - <license> - <name>The Apache License, Version 2.0</name> - <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> - </license> - </licenses> - <developers> - <developer> - <name>Vespa</name> - <url>https://github.com/vespa-engine</url> - </developer> - </developers> - <scm> - <connection>scm:git:git@github.com:vespa-engine/vespa.git</connection> - <developerConnection>scm:git:git@github.com:vespa-engine/vespa.git</developerConnection> - <url>git@github.com:vespa-engine/vespa.git</url> - </scm> - - <properties> - <vespaversion>${project.version}</vespaversion> - <!-- when <test-framework.version> is "${project.version}", it is decoupled from the - compile version set with "-D vespaversion=...", and is instead the newest version. - When it is "${vespaversion}", it is coupled, which should be the default. --> - <test-framework.version>${vespaversion}</test-framework.version> - <target_jdk_version>17</target_jdk_version> - <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> - <maven-dependency-plugin.version>3.3.0</maven-dependency-plugin.version> - <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> - <junit.version>5.8.1</junit.version> <!-- Keep in sync with hosted-tenant-base and tenant-cd until all direct use is removed --> - <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> - <test.categories>!integration</test.categories> - </properties> - - <dependencyManagement> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dependency-versions</artifactId> - <version>${vespaversion}</version> - <type>pom</type> - <scope>import</scope> - </dependency> - - <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - <version>${junit.version}</version> - </dependency> - - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <version>${junit.version}</version> - </dependency> - </dependencies> - </dependencyManagement> - - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container</artifactId> - <version>${vespaversion}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> - <version>${vespaversion}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd-api</artifactId> - <version>${test-framework.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>cloud-tenant-cd</artifactId> - <version>${test-framework.version}</version> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - - <profiles> - <profile> - <!-- Build *-fat-test.jar file that includes all non-test classes and resources - that are part of the class path during test and and test.jar that includes - all test classes and resources, and put it inside a zip: - 1. application classes and resources - 2. test classes and resources - 3. classes and resources in all dependencies of both (1) and (2) - 4. copy the fat-test-jar and test-jar to application-test/artifacts directory - 5. zip application-test --> - <id>fat-test-application</id> - <build> - <plugins> - <plugin> - <!-- dependencies, see (3) above --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <version>3.3.0</version> - <executions> - <execution> - <!-- JAR-like dependencies --> - <id>unpack-dependencies</id> - <phase>prepare-package</phase> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <configuration> - <includeTypes>jar,test-jar</includeTypes> - <!-- felix is not needed in the fat jar --> - <excludeGroupIds>org.apache.felix</excludeGroupIds> - <outputDirectory>target/fat-test-classes</outputDirectory> - <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and - bcprov-jdk15on-1.58.jar are pulled in via - container-dev and both contains the same set of - bouncycastle signature files in META-INF: - BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and - BC2048KE.SF. By merging any of these two with any - other JAR file like we're doing here, the signatures - are wrong. Worse, what we're doing is WRONG but not - yet fatal. - - The symptom of this happening is that the tester fails - to load the SystemTest class(!?), and subsequently - tries to run all test-like files in the fat test JAR. - - The solution is to exclude such files. This happens - automatically with maven-assembly-plugin. --> - <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> - </configuration> - </execution> - <execution> - <!-- non-JAR-like dependencies --> - <id>non-jar-dependencies</id> - <phase>prepare-package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - <configuration> - <excludeTypes>jar,test-jar</excludeTypes> - <outputDirectory>target/fat-test-classes</outputDirectory> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <artifactId>maven-resources-plugin</artifactId> - <version>3.1.0</version> - <executions> - <execution> - <id>copy-resources</id> - <phase>prepare-package</phase> - <goals> - <goal>copy-resources</goal> - </goals> - <configuration> - <outputDirectory>target/fat-test-classes</outputDirectory> - <resources> - <!-- application classes and resources, see 1. above --> - <resource> - <directory>target/classes</directory> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>3.1.0</version> - <executions> - <execution> - <id>fat-test-jar</id> - <phase>package</phase> - <goals> - <goal>jar</goal> - </goals> - <configuration> - <classesDirectory>target/fat-test-classes</classesDirectory> - <classifier>fat-test</classifier> - </configuration> - </execution> - <execution> - <id>test-jar</id> - <phase>package</phase> - <goals> - <goal>test-jar</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>attach-artifact</id> - <phase>package</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <tasks> - <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> - <copy file="target/${project.artifactId}-fat-test.jar" - todir="target/application-test/artifacts/" /> - - <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> - <copy file="target/${project.artifactId}-tests.jar" - todir="target/application-test/artifacts/" /> - - <!-- zip application-test, see 5. above --> - <zip destfile="target/application-test.zip" - basedir="target/application-test/" /> - </tasks> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - - <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> - <id>set-vespa-compile-version</id> - <activation> - <property> - <name>vespa.compile.version</name> - </property> - </activation> - <properties> - <vespaversion>${vespa.compile.version}</vespaversion> - </properties> - </profile> - - <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> - <id>set-vespa-runtime-version</id> - <activation> - <property> - <name>vespa.runtime.version</name> - </property> - </activation> - <properties> - <vespaVersion>${vespa.runtime.version}</vespaVersion> - </properties> - </profile> - <profile> - <id>ossrh-deploy-vespa</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <maven.deploy.skip>true</maven.deploy.skip> - <maven.javadoc.skip>true</maven.javadoc.skip> - <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo> - </properties> - </profile> - </profiles> - - <build> - <finalName>${project.artifactId}</finalName> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <version>${maven-dependency-plugin.version}</version> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <groups>${test.categories}</groups> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - <systemPropertyVariables> - <application>${application}</application> - <tenant>${tenant}</tenant> - <instance>${instance}</instance> - <environment>${environment}</environment> - <region>${region}</region> - <endpoint>${endpoint}</endpoint> - <apiKeyFile>${apiKeyFile}</apiKeyFile> - <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> - <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> - <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> - </systemPropertyVariables> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - </configuration> - </plugin> - </plugins> - </pluginManagement> - - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <version>3.0.0</version> - <executions> - <execution> - <id>enforce-java</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireJavaVersion> - <version>[17, )</version> - </requireJavaVersion> - <requireMavenVersion> - <version>[3.6, )</version> - </requireMavenVersion> - </rules> - </configuration> - </execution> - <execution> - <id>enforce-no-log4j</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <bannedDependencies> - <!-- Fail validation for apps with log4j deps in compile or provided scope. --> - <excludes> - <exclude>log4j:log4j:*:jar:compile</exclude> - <exclude>log4j:log4j:*:jar:provided</exclude> - <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:compile</exclude> - <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:provided</exclude> - </excludes> - </bannedDependencies> - </rules> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>${maven-compiler-plugin.version}</version> - <configuration> - <release>${target_jdk_version}</release> - <showWarnings>true</showWarnings> - <showDeprecation>true</showDeprecation> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-maven-plugin</artifactId> - <version>${vespaversion}</version> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-application-maven-plugin</artifactId> - <version>${vespaversion}</version> - <executions> - <execution> - <goals> - <goal>packageApplication</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${vespaversion}</version> - <extensions>true</extensions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java index bd1511acdbd..415c4ffe8f0 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java @@ -67,7 +67,7 @@ public class DistributionTestCase { public void testSimple() { test = new DistributionTestFactory("simple"); List<BucketId> buckets = getTestBuckets(); - Integer nodes[] = { 6, 3, 4, 8, 8, 8, 8, 8, 8, 3 }; + Integer [] nodes = { 6, 3, 4, 8, 8, 8, 8, 8, 8, 3 }; for (int i=0; i<buckets.size(); ++i) { BucketId bucket = buckets.get(i); DistributionTestFactory.Test t = test.recordResult(bucket).assertNodeCount(1); @@ -137,86 +137,89 @@ public class DistributionTestCase { public void testWriteDistribution() throws IOException, ParseException, Distribution.TooFewBucketBitsInUseException, Distribution.NoDistributorsAvailableException { String clusterState = "distributor:9"; String distributionConfig = - "redundancy 3\n" + - "group[4]\n" + - "group[0].index \"invalid\"\n" + - "group[0].name \"invalid\"\n" + - "group[0].partitions 1|2|*\n" + - "group[0].nodes[0]\n" + - "group[1].index 1\n" + - "group[1].capacity 2.0\n" + - "group[1].name group1\n" + - "group[1].partitions *\n" + - "group[1].nodes[3]\n" + - "group[1].nodes[0].index 0\n" + - "group[1].nodes[1].index 1\n" + - "group[1].nodes[2].index 2\n" + - "group[2].index 2\n" + - "group[2].capacity 3.0\n" + - "group[2].name group2\n" + - "group[2].partitions *\n" + - "group[2].nodes[3]\n" + - "group[2].nodes[0].index 3\n" + - "group[2].nodes[1].index 4\n" + - "group[2].nodes[2].index 5\n" + - "group[3].index 3\n" + - "group[3].capacity 5.0\n" + - "group[3].name group3\n" + - "group[3].partitions *\n" + - "group[3].nodes[3]\n" + - "group[3].nodes[0].index 6\n" + - "group[3].nodes[1].index 7\n" + - "group[3].nodes[2].index 8\n"; + """ + redundancy 3 + group[4] + group[0].index "invalid" + group[0].name "invalid" + group[0].partitions 1|2|* + group[0].nodes[0] + group[1].index 1 + group[1].capacity 2.0 + group[1].name group1 + group[1].partitions * + group[1].nodes[3] + group[1].nodes[0].index 0 + group[1].nodes[1].index 1 + group[1].nodes[2].index 2 + group[2].index 2 + group[2].capacity 3.0 + group[2].name group2 + group[2].partitions * + group[2].nodes[3] + group[2].nodes[0].index 3 + group[2].nodes[1].index 4 + group[2].nodes[2].index 5 + group[3].index 3 + group[3].capacity 5.0 + group[3].name group3 + group[3].partitions * + group[3].nodes[3] + group[3].nodes[0].index 6 + group[3].nodes[1].index 7 + group[3].nodes[2].index 8 + """; writeDistributionTest("depth2", clusterState, distributionConfig); clusterState = "distributor:20 storage:20"; String complexDistributionConfig = - "redundancy 5\n" + - "group[7]\n" + - "group[0].partitions \"*|*\"\n" + - "group[0].index \"invalid\"\n" + - "group[0].name \"invalid\"\n" + - "group[0].nodes[0]\n" + - "group[1].partitions \"1|*\"\n" + - "group[1].index \"0\"\n" + - "group[1].name \"switch0\"\n" + - "group[1].nodes[0]\n" + - "group[2].partitions \"\"\n" + - "group[2].index \"0.0\"\n" + - "group[2].name \"rack0\"\n" + - "group[2].nodes[4]\n" + - "group[2].nodes[0].index 0\n" + - "group[2].nodes[1].index 1\n" + - "group[2].nodes[2].index 2\n" + - "group[2].nodes[3].index 3\n" + - "group[3].partitions \"\"\n" + - "group[3].index \"0.1\"\n" + - "group[3].name \"rack1\"\n" + - "group[3].nodes[4]\n" + - "group[3].nodes[0].index 8\n" + - "group[3].nodes[1].index 9\n" + - "group[3].nodes[2].index 14\n" + - "group[3].nodes[3].index 15\n" + - "group[4].partitions \"*\"\n" + - "group[4].index \"1\"\n" + - "group[4].name \"switch1\"\n" + - "group[4].nodes[0]\n" + - "group[5].partitions \"\"\n" + - "group[5].index \"1.0\"\n" + - "group[5].name \"rack0\"\n" + - "group[5].nodes[4]\n" + - "group[5].nodes[0].index 4\n" + - "group[5].nodes[1].index 5\n" + - "group[5].nodes[2].index 6\n" + - "group[5].nodes[3].index 17\n" + - "group[6].partitions \"\"\n" + - "group[6].index \"1.1\"\n" + - "group[6].name \"rack1\"\n" + - "group[6].nodes[4]\n" + - "group[6].nodes[0].index 10\n" + - "group[6].nodes[1].index 12\n" + - "group[6].nodes[2].index 13\n" + - "group[6].nodes[3].index 7"; + """ + redundancy 5 + group[7] + group[0].partitions "*|*" + group[0].index "invalid" + group[0].name "invalid" + group[0].nodes[0] + group[1].partitions "1|*" + group[1].index "0" + group[1].name "switch0" + group[1].nodes[0] + group[2].partitions "" + group[2].index "0.0" + group[2].name "rack0" + group[2].nodes[4] + group[2].nodes[0].index 0 + group[2].nodes[1].index 1 + group[2].nodes[2].index 2 + group[2].nodes[3].index 3 + group[3].partitions "" + group[3].index "0.1" + group[3].name "rack1" + group[3].nodes[4] + group[3].nodes[0].index 8 + group[3].nodes[1].index 9 + group[3].nodes[2].index 14 + group[3].nodes[3].index 15 + group[4].partitions "*" + group[4].index "1" + group[4].name "switch1" + group[4].nodes[0] + group[5].partitions "" + group[5].index "1.0" + group[5].name "rack0" + group[5].nodes[4] + group[5].nodes[0].index 4 + group[5].nodes[1].index 5 + group[5].nodes[2].index 6 + group[5].nodes[3].index 17 + group[6].partitions "" + group[6].index "1.1" + group[6].name "rack1" + group[6].nodes[4] + group[6].nodes[0].index 10 + group[6].nodes[1].index 12 + group[6].nodes[2].index 13 + group[6].nodes[3].index 7"""; writeDistributionTest("depth3", clusterState, complexDistributionConfig); clusterState = "distributor:20 storage:20 .3.c:3 .7.c:2.5 .12.c:1.5"; @@ -269,7 +272,7 @@ public class DistributionTestCase { buckets.add(new BucketId(distbits, i)); } for (BucketId bucket : buckets) { - DistributionTestFactory.Test t = test.recordResult(bucket).assertNodeCount(1); + test.recordResult(bucket).assertNodeCount(1); } test.recordTestResults(); test = null; @@ -347,7 +350,7 @@ public class DistributionTestCase { .setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3).distributor_auto_ownership_transfer_on_whole_group_down(false)) .setNodeType(NodeType.DISTRIBUTOR) .setClusterState(new ClusterState("distributor:2 storage:9")); - int counts[] = new int[10]; + int [] counts = new int[10]; int noneExisting = 0; for (BucketId bucket : getTestBuckets()) { DistributionTestFactory.Test t = test.recordResult(bucket); @@ -393,7 +396,7 @@ public class DistributionTestCase { test = new DistributionTestFactory("group-capacity") .setNodeCount(getNodeCount(1, 3, 3)).setDistribution(config); - int counts[] = new int[9]; + int [] counts = new int[9]; for (int i=0; i<900; ++i) { BucketId bucket = new BucketId(16, i); ++counts[ test.recordResult(bucket).assertNodeCount(1).getNodes().get(0) ]; diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 461e3f0aef5..31d547807d9 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -6,8 +6,6 @@ ai.djl:api:0.20.0 ai.djl.huggingface:tokenizers:0.20.0 aopalliance:aopalliance:1.0 backport-util-concurrent:backport-util-concurrent:3.1 -biz.aQute.bnd:biz.aQute.bnd.util:6.1.0 -biz.aQute.bnd:biz.aQute.bndlib:6.1.0 ch.qos.logback:logback-classic:1.2.10 ch.qos.logback:logback-core:1.2.10 classworlds:classworlds:1.1-alpha-2 @@ -81,8 +79,8 @@ javax.xml.bind:jaxb-api:2.3.0 joda-time:joda-time:2.12.2 net.java.dev.jna:jna:5.11.0 net.openhft:zero-allocation-hashing:0.16 -org.antlr:antlr-runtime:3.5.2 -org.antlr:antlr4-runtime:4.9.3 +org.antlr:antlr-runtime:3.5.3 +org.antlr:antlr4-runtime:4.11.1 org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.5 org.apache.commons:commons-compress:1.22 org.apache.commons:commons-csv:1.8 @@ -101,21 +99,21 @@ org.apache.httpcomponents.client5:httpclient5:5.2.1 org.apache.httpcomponents.core5:httpcore5:5.2.1 org.apache.httpcomponents.core5:httpcore5-h2:5.2.1 org.apache.maven:maven-archiver:3.6.0 -org.apache.maven:maven-artifact:3.8.6 +org.apache.maven:maven-artifact:3.8.7 org.apache.maven:maven-artifact-manager:2.2.1 -org.apache.maven:maven-builder-support:3.8.6 +org.apache.maven:maven-builder-support:3.8.7 org.apache.maven:maven-compat:3.0 -org.apache.maven:maven-core:3.8.6 -org.apache.maven:maven-model:3.8.6 -org.apache.maven:maven-model-builder:3.8.6 -org.apache.maven:maven-plugin-api:3.8.6 +org.apache.maven:maven-core:3.8.7 +org.apache.maven:maven-model:3.8.7 +org.apache.maven:maven-model-builder:3.8.7 +org.apache.maven:maven-plugin-api:3.8.7 org.apache.maven:maven-plugin-registry:2.2.1 org.apache.maven:maven-profile:2.2.1 org.apache.maven:maven-project:2.2.1 -org.apache.maven:maven-repository-metadata:3.8.6 -org.apache.maven:maven-resolver-provider:3.8.6 -org.apache.maven:maven-settings:3.8.6 -org.apache.maven:maven-settings-builder:3.8.6 +org.apache.maven:maven-repository-metadata:3.8.7 +org.apache.maven:maven-resolver-provider:3.8.7 +org.apache.maven:maven-settings:3.8.7 +org.apache.maven:maven-settings-builder:3.8.7 org.apache.maven.plugin-tools:maven-plugin-annotations:3.6.4 org.apache.maven.plugins:maven-jar-plugin:3.2.0 org.apache.maven.resolver:maven-resolver-api:1.6.3 diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java index 8b0509dc788..3192bb4f225 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/ApacheCluster.java @@ -5,11 +5,9 @@ import ai.vespa.feed.client.FeedClientBuilder.Compression; import ai.vespa.feed.client.HttpResponse; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; -import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; import org.apache.hc.client5.http.impl.async.HttpAsyncClients; -import org.apache.hc.client5.http.impl.async.MinimalH2AsyncClient; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.core5.concurrent.FutureCallback; import org.apache.hc.core5.http.ContentType; @@ -60,10 +58,9 @@ class ApacheCluster implements Cluster { private final ScheduledExecutorService timeoutExecutor = Executors.newSingleThreadScheduledExecutor(t -> new Thread(t, "request-timeout-thread")); ApacheCluster(FeedClientBuilderImpl builder) throws IOException { - ConnectionConfig connectionConfig = createConnectionConfig(); for (int i = 0; i < builder.connectionsPerEndpoint; i++) for (URI endpoint : builder.endpoints) - endpoints.add(new Endpoint(createHttpClient(builder, connectionConfig), endpoint)); + endpoints.add(new Endpoint(createHttpClient(builder), endpoint)); this.requestConfig = createRequestConfig(builder); this.compression = builder.compression; } @@ -163,7 +160,7 @@ class ApacheCluster implements Cluster { } @SuppressWarnings("deprecation") - private static CloseableHttpAsyncClient createHttpClient(FeedClientBuilderImpl builder, ConnectionConfig connectionConfig) throws IOException { + private static CloseableHttpAsyncClient createHttpClient(FeedClientBuilderImpl builder) throws IOException { SSLContext sslContext = builder.constructSslContext(); String[] allowedCiphers = excludeH2Blacklisted(excludeWeak(sslContext.getSupportedSSLParameters().getCipherSuites())); if (allowedCiphers.length == 0) @@ -176,20 +173,18 @@ class ApacheCluster implements Cluster { if (builder.hostnameVerifier != null) tlsStrategyBuilder.setHostnameVerifier(builder.hostnameVerifier); - MinimalH2AsyncClient client = HttpAsyncClients.createHttp2Minimal(H2Config.custom() + return HttpAsyncClients.createHttp2Minimal(H2Config.custom() .setMaxConcurrentStreams(builder.maxStreamsPerConnection) .setCompressionEnabled(true) .setPushEnabled(false) .setInitialWindowSize(Integer.MAX_VALUE) .build(), IOReactorConfig.custom() - .setIoThreadCount(2) - .setTcpNoDelay(true) - .setSoTimeout(Timeout.ofSeconds(10)) - .build(), + .setIoThreadCount(2) + .setTcpNoDelay(true) + .setSoTimeout(Timeout.ofSeconds(10)) + .build(), tlsStrategyBuilder.build()); - client.setConnectionConfigResolver(host -> connectionConfig); - return client; } private static int portOf(URI url) { @@ -197,19 +192,13 @@ class ApacheCluster implements Cluster { : url.getPort(); } - private static ConnectionConfig createConnectionConfig() { - return ConnectionConfig.custom() - .setConnectTimeout(Timeout.ofSeconds(10)).build(); - } - @SuppressWarnings("deprecation") private static RequestConfig createRequestConfig(FeedClientBuilderImpl b) { RequestConfig.Builder builder = RequestConfig.custom() + .setConnectTimeout(Timeout.ofSeconds(10)) .setConnectionRequestTimeout(Timeout.DISABLED) .setResponseTimeout(Timeout.ofSeconds(190)); - if (b.proxy != null) { - builder.setProxy(new HttpHost(b.proxy.getScheme(), b.proxy.getHost(), b.proxy.getPort())); - } + if (b.proxy != null) builder.setProxy(new HttpHost(b.proxy.getScheme(), b.proxy.getHost(), b.proxy.getPort())); return builder.build(); } diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java index 30ed8dcfdd4..9195b5ab858 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java @@ -48,7 +48,7 @@ class ApacheClusterTest { Map.of("name1", () -> "value1", "name2", () -> "value2"), "content".getBytes(UTF_8), - Duration.ofSeconds(10)), + Duration.ofSeconds(20)), vessel); HttpResponse response = vessel.get(15, TimeUnit.SECONDS); assertEquals("{}", new String(response.body(), UTF_8)); diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 498827e839b..c3b87278345 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -976,6 +976,7 @@ "public com.yahoo.tensor.Tensor$Builder cell(com.yahoo.tensor.TensorAddress, double)", "public com.yahoo.tensor.Tensor$Builder block(com.yahoo.tensor.TensorAddress, double[])", "public com.yahoo.tensor.MixedTensor build()", + "public static com.yahoo.tensor.MixedTensor$BoundBuilder of(com.yahoo.tensor.TensorType)", "public bridge synthetic com.yahoo.tensor.Tensor build()" ], "fields" : [ ] @@ -1026,6 +1027,7 @@ "public com.yahoo.tensor.MixedTensor build()", "public void trackBounds(com.yahoo.tensor.TensorAddress)", "public com.yahoo.tensor.TensorType createBoundType()", + "public static com.yahoo.tensor.MixedTensor$UnboundBuilder of(com.yahoo.tensor.TensorType)", "public bridge synthetic com.yahoo.tensor.Tensor build()" ], "fields" : [ ] @@ -1466,6 +1468,7 @@ "public static com.yahoo.tensor.TensorType fromSpec(java.lang.String)", "public com.yahoo.tensor.TensorType$Value valueType()", "public com.yahoo.tensor.TensorType mappedSubtype()", + "public com.yahoo.tensor.TensorType indexedSubtype()", "public int rank()", "public java.util.List dimensions()", "public java.util.Set dimensionNames()", diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java deleted file mode 100644 index 4b7d2ba4094..00000000000 --- a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import java.nio.channels.WritableByteChannel; -import java.io.IOException; -import java.io.OutputStream; -import java.util.Stack; -import java.util.LinkedList; -import java.util.Iterator; -import java.nio.ByteBuffer; - -/** - * @author Bjørn Borud - */ -public class GrowableBufferOutputStream extends OutputStream { - - private ByteBuffer lastBuffer; - private final ByteBuffer directBuffer; - private final LinkedList<ByteBuffer> bufferList = new LinkedList<>(); - private final Stack<ByteBuffer> recycledBuffers = new Stack<>(); - - private final int bufferSize; - private final int maxBuffers; - - public GrowableBufferOutputStream(int bufferSize, int maxBuffers) { - this.bufferSize = bufferSize; - this.maxBuffers = maxBuffers; - lastBuffer = ByteBuffer.allocate(bufferSize); - directBuffer = ByteBuffer.allocateDirect(bufferSize); - } - - @Override - public void write(byte[] cbuf, int off, int len) throws IOException { - if (lastBuffer.remaining() >= len) { - lastBuffer.put(cbuf, off, len); - return; - } - - int residue = len; - - while (residue > 0) { - int newOffset = len - residue; - int toWrite = Math.min(lastBuffer.remaining(), residue); - - lastBuffer.put(cbuf, newOffset, toWrite); - residue -= toWrite; - if (residue != 0) { - extend(); - } - } - } - - @Override - public void write(byte[] b) throws IOException { - write(b,0,b.length); - } - - @Override - public String toString() { - return "GrowableBufferOutputStream, writable size " + writableSize() - + " bytes, " + numWritableBuffers() + " buffers, last buffer" - + " position " + lastBuffer.position() + ", last buffer limit " - + lastBuffer.limit(); - } - - public void write(int b) { - if (lastBuffer.remaining() == 0) { - extend(); - } - lastBuffer.put((byte) b); - } - - @Override - public void flush() { - // if the last buffer is untouched we do not need to do anything; if - // it has been touched we call extend(), which enqueues the buffer - // and allocates or recycles a buffer for us - if (lastBuffer.position() > 0) { - extend(); - } - } - - @Override - public void close() { - flush(); - } - - public int channelWrite(WritableByteChannel channel) throws IOException { - ByteBuffer buffer; - int totalWritten = 0; - - while (!bufferList.isEmpty()) { - buffer = bufferList.getFirst(); - int written = 0; - - synchronized (directBuffer) { - directBuffer.clear(); - directBuffer.put(buffer); - directBuffer.flip(); - written = channel.write(directBuffer); - int left = directBuffer.remaining(); - - if (left > 0) { - int oldpos = buffer.position(); - - buffer.position(oldpos - left); - } - totalWritten += written; - } - - // if we've completed writing this buffer we can dispose of it - if (buffer.remaining() == 0) { - bufferList.removeFirst(); - recycleBuffer(buffer); - } - - // if we didn't write any bytes we terminate - if (written == 0) { - break; - } - } - - return totalWritten; - } - - public int numWritableBuffers() { - return bufferList.size(); - } - - public void clear() { - flush(); - bufferList.clear(); - } - - public void clearCache() { - recycledBuffers.clear(); - } - - public void clearAll() { - clear(); - clearCache(); - } - - public int writableSize() { - Iterator<ByteBuffer> it = bufferList.iterator(); - int size = 0; - - while (it.hasNext()) { - size += (it.next()).remaining(); - } - - return size; - } - - public ByteBuffer[] getWritableBuffers() { - flush(); - ByteBuffer[] result = new ByteBuffer[numWritableBuffers()]; - return bufferList.toArray(result); - } - - private void extend() { - enqueueBuffer(lastBuffer); - - if (recycledBuffers.empty()) { - lastBuffer = ByteBuffer.allocate(bufferSize); - } else { - lastBuffer = recycledBuffers.pop(); - lastBuffer.clear(); - } - } - - private void enqueueBuffer(ByteBuffer buffer) { - buffer.flip(); - bufferList.addLast(buffer); - } - - private void recycleBuffer(ByteBuffer buffer) { - if (recycledBuffers.size() >= maxBuffers) { - return; - } - recycledBuffers.push(buffer); - } - -} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 2027dcfb60f..33e83c00e74 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -305,6 +305,10 @@ public class MixedTensor implements Tensor { return new MixedTensor(type, builder, indexBuilder.build()); } + public static BoundBuilder of(TensorType type) { + return new BoundBuilder(type); + } + } /** @@ -371,6 +375,10 @@ public class MixedTensor implements Tensor { return typeBuilder.build(); } + public static UnboundBuilder of(TensorType type) { + return new UnboundBuilder(type); + } + } /** diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 5636150bca1..d5c3b1340f1 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -13,21 +13,8 @@ import java.util.stream.Collectors; * @author bratseth */ public abstract class TensorAddress implements Comparable<TensorAddress> { - private static final String [] SMALL_INDEXES = createSmallIndexesAsStrings(1000); - private static String [] createSmallIndexesAsStrings(int count) { - String [] asStrings = new String[count]; - for (int i = 0; i < count; i++) { - asStrings[i] = String.valueOf(i); - } - return asStrings; - } - private static String asString(int index) { - return (index < SMALL_INDEXES.length) ? SMALL_INDEXES[index] : String.valueOf(index); - } - private static String asString(long index) { - return (index < SMALL_INDEXES.length) ? SMALL_INDEXES[(int)index] : String.valueOf(index); - } + private static final String [] SMALL_INDEXES = createSmallIndexesAsStrings(1000); public static TensorAddress of(String[] labels) { return new StringTensorAddress(labels); @@ -86,8 +73,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof TensorAddress)) return false; - TensorAddress other = (TensorAddress)o; + if ( ! (o instanceof TensorAddress other)) return false; if (other.size() != this.size()) return false; for (int i = 0; i < this.size(); i++) if ( ! Objects.equals(this.label(i), other.label(i))) @@ -115,6 +101,18 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return "'" + label + "'"; } + private static String[] createSmallIndexesAsStrings(int count) { + String [] asStrings = new String[count]; + for (int i = 0; i < count; i++) { + asStrings[i] = String.valueOf(i); + } + return asStrings; + } + + private static String asString(long index) { + return (index < SMALL_INDEXES.length) ? SMALL_INDEXES[(int)index] : String.valueOf(index); + } + private static final class StringTensorAddress extends TensorAddress { private final String[] labels; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 36693280183..57d276f278e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -88,6 +88,7 @@ public class TensorType { private final List<Dimension> dimensions; private final TensorType mappedSubtype; + private final TensorType indexedSubtype; public TensorType(Value valueType, Collection<Dimension> dimensions) { this.valueType = valueType; @@ -95,12 +96,18 @@ public class TensorType { Collections.sort(dimensionList); this.dimensions = List.copyOf(dimensionList); - if (dimensionList.stream().allMatch(d -> d.isIndexed())) + if (dimensionList.stream().allMatch(d -> d.isIndexed())) { mappedSubtype = empty; - else if (dimensionList.stream().noneMatch(d -> d.isIndexed())) + indexedSubtype = this; + } + else if (dimensionList.stream().noneMatch(d -> d.isIndexed())) { mappedSubtype = this; - else - mappedSubtype = new TensorType(valueType, dimensions.stream().filter(d -> ! d.isIndexed()).toList()); + indexedSubtype = empty; + } + else { + mappedSubtype = new TensorType(valueType, dimensions.stream().filter(d -> !d.isIndexed()).toList()); + indexedSubtype = new TensorType(valueType, dimensions.stream().filter(Dimension::isIndexed).toList()); + } } static public Value combinedValueType(TensorType ... types) { @@ -135,6 +142,9 @@ public class TensorType { /** The type representing the mapped subset of dimensions of this. */ public TensorType mappedSubtype() { return mappedSubtype; } + /** The type representing the indexed subset of dimensions of this. */ + public TensorType indexedSubtype() { return indexedSubtype; } + /** Returns the number of dimensions of this: dimensions().size() */ public int rank() { return dimensions.size(); } diff --git a/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java b/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java deleted file mode 100644 index 9c905979131..00000000000 --- a/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import java.nio.channels.WritableByteChannel; -import java.nio.ByteBuffer; -import java.io.IOException; -import com.yahoo.io.GrowableBufferOutputStream; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - - -/** - * Tests the GrowableBufferOutputStream - * - * @author Bjorn Borud - */ -public class GrowableBufferOutputStreamTestCase { - - private byte[] testData; - - static class DummyWritableByteChannel implements WritableByteChannel { - private ByteBuffer buffer; - - public DummyWritableByteChannel(ByteBuffer buffer) { - this.buffer = buffer; - } - - public int write(ByteBuffer src) { - int written = Math.min(src.remaining(), buffer.remaining()); - - if (buffer.remaining() < src.remaining()) { - ByteBuffer tmp = src.slice(); - - tmp.limit(written); - src.position(src.position() + written); - } else { - buffer.put(src); - } - return written; - } - - public boolean isOpen() { - return true; - } - - public void close() {} - } - - @Before - public void setUp() { - testData = new byte[100]; - for (int i = 0; i < 100; ++i) { - testData[i] = (byte) i; - } - } - - @Test - public void testSimple() throws IOException { - GrowableBufferOutputStream g = new GrowableBufferOutputStream(10, 5); - - g.write(testData, 0, 100); - g.flush(); - assertEquals(10, g.numWritableBuffers()); - assertEquals(100, g.writableSize()); - - ByteBuffer sink = ByteBuffer.allocate(60); - DummyWritableByteChannel channel = new DummyWritableByteChannel(sink); - int written = g.channelWrite(channel); - - assertEquals(60, written); - assertEquals(60, sink.position()); - assertEquals(40, g.writableSize()); - - // there should be 4 buffers left now - assertEquals(4, g.numWritableBuffers()); - - // ensure that we got what we expected - for (int i = 0; i < 60; ++i) { - if (((int) sink.get(i)) != i) { - fail(); - } - } - - // then we write more data - g.write(testData, 0, 100); - g.flush(); - assertEquals(140, g.writableSize()); - - // ...which implies that we should now have 14 writable buffers - assertEquals(14, g.numWritableBuffers()); - - // reset the sink so it can consume more data - sink.clear(); - - // then write more to the DummyWritableByteChannel - written = g.channelWrite(channel); - assertEquals(60, written); - assertEquals(60, sink.position()); - assertEquals(80, g.writableSize()); - - // now there should be 8 buffers - assertEquals(8, g.numWritableBuffers()); - - // ensure that we got what we expected - for (int i = 0; i < 60; ++i) { - int val = (int) sink.get(i); - int expected = (i + 60) % 100; - - if (val != expected) { - fail("Value was " + val + " and not " + i); - } - } - - // when we clear there should be no buffers - g.clear(); - assertEquals(0, g.numWritableBuffers()); - assertEquals(0, g.writableSize()); - - // ditto after flush after clear - g.flush(); - assertEquals(0, g.numWritableBuffers()); - - // flush the cache too - g.clearAll(); - assertEquals(0, g.numWritableBuffers()); - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java index 738697d4521..ba541ab2cd6 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java @@ -102,6 +102,30 @@ public class TensorTypeTestCase { assertEquals("tensor<int8>(x[])", TensorType.fromSpec("tensor<int8>(x[])").toString()); } + @Test + public void testIndexedSubtype() { + assertEquals(TensorType.fromSpec("tensor(x[10])"), + TensorType.fromSpec("tensor(x[10])").indexedSubtype()); + assertEquals(TensorType.fromSpec("tensor(x[10])"), + TensorType.fromSpec("tensor(x[10],a{})").indexedSubtype()); + assertEquals(TensorType.fromSpec("tensor(x[10],y[5])"), + TensorType.fromSpec("tensor(x[10],y[5],a{},b{})").indexedSubtype()); + assertEquals(TensorType.fromSpec("tensor()"), + TensorType.fromSpec("tensor(a{})").indexedSubtype()); + } + + @Test + public void testMappedSubtype() { + assertEquals(TensorType.fromSpec("tensor(a{})"), + TensorType.fromSpec("tensor(a{})").mappedSubtype()); + assertEquals(TensorType.fromSpec("tensor(a{})"), + TensorType.fromSpec("tensor(x[10],a{})").mappedSubtype()); + assertEquals(TensorType.fromSpec("tensor(a{},b{})"), + TensorType.fromSpec("tensor(x[10],y[5],a{},b{})").mappedSubtype()); + assertEquals(TensorType.fromSpec("tensor()"), + TensorType.fromSpec("tensor(x[10])").mappedSubtype()); + } + private static void assertTensorType(String typeSpec) { assertTensorType(typeSpec, typeSpec); } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 8509d5fc382..76308260578 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -50,6 +50,7 @@ vespa_define_module( src/tests/coro/generator src/tests/coro/lazy src/tests/coro/received + src/tests/coro/waiting_for src/tests/cpu_usage src/tests/crc src/tests/crypto diff --git a/vespalib/src/tests/coro/waiting_for/CMakeLists.txt b/vespalib/src/tests/coro/waiting_for/CMakeLists.txt new file mode 100644 index 00000000000..d9eaa7eaf03 --- /dev/null +++ b/vespalib/src/tests/coro/waiting_for/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_waiting_for_test_app TEST + SOURCES + waiting_for_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_waiting_for_test_app COMMAND vespalib_waiting_for_test_app) diff --git a/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp b/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp new file mode 100644 index 00000000000..385d4ad24e3 --- /dev/null +++ b/vespalib/src/tests/coro/waiting_for/waiting_for_test.cpp @@ -0,0 +1,110 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/coro/lazy.h> +#include <vespa/vespalib/coro/completion.h> +#include <vespa/vespalib/coro/waiting_for.h> +#include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::coro; + +struct AsyncService { + std::vector<WaitingFor<int>> pending; + auto get_value() { + return awaiter_for<int>([&](WaitingFor<int> handle) + { + pending.push_back(std::move(handle)); + }); + } +}; + +struct AsyncVoidService { + std::vector<void*> pending; + auto get_value() { + return awaiter_for<int>([&](WaitingFor<int> handle) + { + pending.push_back(handle.release()); + }); + } +}; + +struct SyncService { + auto get_value() { + return awaiter_for<int>([](WaitingFor<int> handle) + { + handle.set_value(42); + return handle.release_waiter(); // symmetric transfer + }); + } +}; + +template<typename Service> +Lazy<int> wait_for_value(Service &service) { + int value = co_await service.get_value(); + co_return value; +} + +template <typename T> +Lazy<T> wait_any(auto &&fun) { + T result = co_await fun(); + co_return std::move(result); +} + +TEST(WaitingForTest, wait_for_external_async_int) { + AsyncService service; + auto res = make_future(wait_for_value(service)); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::timeout); + ASSERT_EQ(service.pending.size(), 1); + service.pending[0].set_value(42); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::timeout); + service.pending.clear(); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::ready); + EXPECT_EQ(res.get(), 42); +} + +TEST(WaitingForTest, wait_for_external_async_int_via_void_ptr) { + AsyncVoidService service; + auto res = make_future(wait_for_value(service)); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::timeout); + ASSERT_EQ(service.pending.size(), 1); + { + auto handle = WaitingFor<int>::from_pointer(service.pending[0]); + handle.set_value(42); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::timeout); + } + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::ready); + EXPECT_EQ(res.get(), 42); +} + +TEST(WaitingForTest, wait_for_external_sync_int) { + SyncService service; + auto res = make_future(wait_for_value(service)); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::ready); + EXPECT_EQ(res.get(), 42); +} + +TEST(WaitingForTest, wait_for_move_only_value) { + auto val = std::make_unique<int>(42); + auto fun = [&val](auto handle){ handle.set_value(std::move(val)); }; // asymmetric transfer + auto res = make_future(wait_any<decltype(val)>([&fun](){ return awaiter_for<decltype(val)>(fun); })); + EXPECT_TRUE(res.wait_for(0ms) == std::future_status::ready); + EXPECT_EQ(*res.get(), 42); +} + +TEST(WaitingForTest, set_error) { + PromiseState<int> state; + WaitingFor<int> pending = WaitingFor<int>::from_state(state); + pending.set_error(std::make_exception_ptr(13)); + EXPECT_TRUE(state.result.has_error()); +} + +TEST(WaitingForTest, set_done) { + PromiseState<int> state; + WaitingFor<int> pending = WaitingFor<int>::from_state(state); + pending.set_value(5); + EXPECT_TRUE(state.result.has_value()); + pending.set_done(); + EXPECT_TRUE(state.result.was_canceled()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp index a7102ec9868..1d666524502 100644 --- a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp +++ b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp @@ -33,7 +33,7 @@ TEST("that caught silenced exception causes exitcode 0") { // setrlimit with RLIMIT_AS is broken on Darwin #else TEST("that mmap within limits are fine cause exitcode 0") { - Process proc("exec ./vespalib_mmap_app 150000000 10485760 1"); + Process proc("exec ./vespalib_mmap_app 536870912 10485760 1"); EXPECT_EQUAL(proc.join(), 0); } diff --git a/vespalib/src/tests/util/bfloat16/CMakeLists.txt b/vespalib/src/tests/util/bfloat16/CMakeLists.txt index fe40bf72a14..fa5a8ae4fe2 100644 --- a/vespalib/src/tests/util/bfloat16/CMakeLists.txt +++ b/vespalib/src/tests/util/bfloat16/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +if(EXISTS /opt/vespa-deps/include/onnxruntime/core/framework/endian.h) vespa_add_executable(vespalib_bfloat16_test_app TEST SOURCES bfloat16_test.cpp @@ -7,3 +8,4 @@ vespa_add_executable(vespalib_bfloat16_test_app TEST GTest::GTest ) vespa_add_test(NAME vespalib_bfloat16_test_app COMMAND vespalib_bfloat16_test_app) +endif() diff --git a/vespalib/src/vespa/vespalib/coro/lazy.h b/vespalib/src/vespa/vespalib/coro/lazy.h index 974968d0c77..17077dccc9f 100644 --- a/vespalib/src/vespa/vespalib/coro/lazy.h +++ b/vespalib/src/vespa/vespalib/coro/lazy.h @@ -2,9 +2,8 @@ #pragma once -#include "received.h" +#include "waiting_for.h" -#include <concepts> #include <coroutine> #include <optional> #include <exception> @@ -27,7 +26,8 @@ namespace vespalib::coro { template <std::movable T> class [[nodiscard]] Lazy { public: - struct promise_type { + struct promise_type final : PromiseState<T> { + using PromiseState<T>::result; Lazy<T> get_return_object() { return Lazy(Handle::from_promise(*this)); } static std::suspend_always initial_suspend() noexcept { return {}; } static auto final_suspend() noexcept { @@ -47,11 +47,7 @@ public: void unhandled_exception() noexcept { result.set_error(std::current_exception()); } - Received<T> result; - std::coroutine_handle<> waiter; - promise_type(promise_type &&) = delete; - promise_type(const promise_type &) = delete; - promise_type() noexcept : result(), waiter(std::noop_coroutine()) {} + promise_type() noexcept : PromiseState<T>() {} ~promise_type(); }; using Handle = std::coroutine_handle<promise_type>; diff --git a/vespalib/src/vespa/vespalib/coro/received.h b/vespalib/src/vespa/vespalib/coro/received.h index abc66cd2a9d..305a187249c 100644 --- a/vespalib/src/vespa/vespalib/coro/received.h +++ b/vespalib/src/vespa/vespalib/coro/received.h @@ -3,6 +3,7 @@ #pragma once #include <memory> +#include <concepts> #include <variant> #include <exception> #include <stdexcept> @@ -46,8 +47,8 @@ private: } public: Received() : _value() {} - void set_value(T &&value) { _value.template emplace<1>(std::move(value)); } - void set_value(const T &value) { _value.template emplace<1>(value); } + template <typename RET> + void set_value(RET &&value) { _value.template emplace<1>(std::forward<RET>(value)); } void set_error(std::exception_ptr exception) { _value.template emplace<0>(exception); } void set_done() { _value.template emplace<0>(nullptr); } bool has_value() const { return (_value.index() == 1); } diff --git a/vespalib/src/vespa/vespalib/coro/waiting_for.h b/vespalib/src/vespa/vespalib/coro/waiting_for.h new file mode 100644 index 00000000000..2e11a9cb38c --- /dev/null +++ b/vespalib/src/vespa/vespalib/coro/waiting_for.h @@ -0,0 +1,108 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "received.h" +#include <coroutine> +#include <utility> + +namespace vespalib::coro { + +// State representing that someone (waiter) is waiting for something +// (result). This object cannot be moved or copied. +template <typename T> +struct PromiseState { + Received<T> result; + std::coroutine_handle<> waiter; + PromiseState(const PromiseState &) = delete; + PromiseState &operator=(const PromiseState &) = delete; + PromiseState(PromiseState &&) = delete; + PromiseState &operator=(PromiseState &&) = delete; + PromiseState() noexcept : result(), waiter(std::noop_coroutine()) {} + ~PromiseState(); +}; +template <typename T> +PromiseState<T>::~PromiseState() = default; + +// A thin (smart) wrapper referencing a PromiseState<T> representing +// that a coroutine is waiting for a value. This class acts as a +// receiver in order to set the result value. When the owning +// reference is deleted, the waiting coroutine will be resumed. +template <typename T> +class WaitingFor { +private: + PromiseState<T> *_state; + WaitingFor(PromiseState<T> *state) noexcept : _state(state) {} +public: + WaitingFor(WaitingFor &&rhs) noexcept : _state(std::exchange(rhs._state, nullptr)) {} + WaitingFor(WaitingFor &rhs) = delete; + WaitingFor &operator=(WaitingFor &rhs) = delete; + ~WaitingFor(); + template <typename RET> + void set_value(RET &&value) { + _state->result.set_value(std::forward<RET>(value)); + } + void set_error(std::exception_ptr exception) { + _state->result.set_error(exception); + } + void set_done() { + _state->result.set_done(); + } + std::coroutine_handle<> release_waiter() { + return std::exchange(_state->waiter, std::noop_coroutine()); + } + void *release() { + return std::exchange(_state, nullptr); + } + static WaitingFor from_pointer(void *ptr) { + PromiseState<T> *state = reinterpret_cast<PromiseState<T>*>(ptr); + return {state}; + } + static WaitingFor from_state(PromiseState<T> &state) { + return {&state}; + } +}; + +template <typename T> +WaitingFor<T>::~WaitingFor() +{ + if (_state != nullptr) { + _state->waiter.resume(); + } +} + +static_assert(receiver_of<WaitingFor<int>, int>); +static_assert(receiver_of<WaitingFor<std::unique_ptr<int>>, std::unique_ptr<int>>); + +// Create a custom awaiter that will return a value of type T when the +// coroutine is resumed. The waiting coroutine will be represented as +// a WaitingFor<T> that is passed as the only parameter to 'f'. The +// return value of 'f' is returned from await_suspend, which means it +// must be void, bool or coroutine handle. If 'f' returns a value +// indicating that the coroutine should be resumed immediately, +// WaitingFor<T>::release_waiter() must be called to avoid resume +// being called as well. Note that await_ready will always return +// false, since the coroutine needs to be suspended in order to create +// the WaitingFor<T> object needed. Also, the WaitingFor<T> api +// implies that the value will be set from the outside and thus cannot +// be ready up-front. Also note that await_resume must return T by +// value, since the awaiter containing the result is a temporary +// object. +template <typename T, typename F> +auto awaiter_for(F &&f) { + struct awaiter final : PromiseState<T> { + using PromiseState<T>::result; + using PromiseState<T>::waiter; + std::decay_t<F> fun; + awaiter(F &&f) : PromiseState<T>(), fun(std::forward<F>(f)) {} + bool await_ready() const noexcept { return false; } + T await_resume() { return std::move(result).get_value(); } + decltype(auto) await_suspend(std::coroutine_handle<> handle) __attribute__((noinline)) { + waiter = handle; + return fun(WaitingFor<T>::from_state(*this)); + } + }; + return awaiter(std::forward<F>(f)); +} + +} |