From 37c1c53f78e5e7f2277220482f201e4b39456345 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 20 Sep 2023 11:41:46 +0200 Subject: Remove unused constructor --- .../com/yahoo/vespa/model/container/ContainerModelEvaluation.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 906ef739ef1..1b47f59653e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -45,10 +45,6 @@ public class ContainerModelEvaluation implements private final RankProfileList rankProfileList; private final FileDistributedOnnxModels onnxModels; // For cluster specific ONNX model settings - public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { - this(cluster, rankProfileList, null); - } - public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList, FileDistributedOnnxModels onnxModels) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); -- cgit v1.2.3 From 8be4b5589dd0744e9e21acfb29347624c4871604 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 20 Sep 2023 14:33:02 +0200 Subject: Add `getSize()` to `ApplicationFile` --- .../model/application/provider/FilesApplicationFile.java | 15 ++++++++++++--- .../com/yahoo/config/application/api/ApplicationFile.java | 2 ++ .../yahoo/config/model/test/MockApplicationPackage.java | 2 ++ .../vespa/config/server/zookeeper/ZKApplicationFile.java | 9 +++++---- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 6b0adb5d079..2dbbc8a5820 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -5,13 +5,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import java.util.logging.Level; -import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.yolean.Exceptions; -import java.io.*; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -208,6 +215,8 @@ public class FilesApplicationFile extends ApplicationFile { } } + @Override public long getSize() { return file.length(); } + @Override public int compareTo(ApplicationFile other) { if (other == this) return 0; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index a55ae795d28..97336b2bca0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -160,6 +160,8 @@ public abstract class ApplicationFile implements Comparable { public abstract MetaData getMetaData(); + public abstract long getSize(); + public static class MetaData { public String status = "unknown"; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index dbcd1cea2fa..342b5f243e7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -488,6 +488,8 @@ public class MockApplicationPackage implements ApplicationPackage { throw new UnsupportedOperationException(); } + @Override public long getSize() { return file.length(); } + @Override public int compareTo(ApplicationFile other) { return this.getPath().getName().compareTo((other).getPath().getName()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 6bc29331efb..13bf4b2368b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -3,8 +3,9 @@ package com.yahoo.vespa.config.server.zookeeper; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.path.Path; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.util.ConfigUtils; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; @@ -13,11 +14,9 @@ import java.io.InputStream; import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; -import java.util.logging.Level; -import com.yahoo.vespa.config.util.ConfigUtils; - import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH; @@ -184,6 +183,8 @@ class ZKApplicationFile extends ApplicationFile { } } + @Override public long getSize() { return zkApp.getBytes(getZKPath(path)).length; } + @Override public int compareTo(ApplicationFile other) { if (other == this) return 0; -- cgit v1.2.3 From b0e3c533424a95b2086bd7db1ca3e0a7a591e64f Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 21 Sep 2023 09:30:18 +0200 Subject: Add overloads taking supplier and throwable --- .../com/yahoo/config/application/api/DeployLogger.java | 5 +++++ .../vespa/config/server/deploy/DeployHandlerLogger.java | 15 +++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java index d9ebd902e3e..65e6bc2803a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeployLogger.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; +import java.util.function.Supplier; import java.util.logging.Level; /** @@ -13,6 +14,10 @@ public interface DeployLogger { /** Log a message unrelated to the application package, e.g. internal error/status. */ void log(Level level, String message); + default void log(Level level, Supplier message) { log(level, message.get()); } + + default void log(Level level, Supplier message, Throwable throwable) { log(level, message); } + /** * Log a message related to the application package. These messages should be actionable by the user, f.ex. to * signal usage of invalid/deprecated syntax diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index 154d2d0f2f0..042aa2423f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -11,6 +11,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -35,15 +36,17 @@ public class DeployHandlerLogger implements DeployLogger { this.logroot = slime.setObject().setArray("log"); } + @Override public void log(Level level, String message) { log(level, () -> message); } + @Override public void log(Level level, Supplier message) { log(level, message, null); } + @Override @SuppressWarnings("deprecation") - public void log(Level level, String message) { - if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) - return; + public void log(Level level, Supplier supplier, Throwable throwable) { + // Also tee to a normal log, Vespa log for example, but use level fine + log.log(Level.FINE, throwable, () -> prefix + supplier.get()); - logJson(level, message); - // Also tee to a normal log, Vespa log for example, but use level fine - log.log(Level.FINE, () -> prefix + message); + if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) return; + logJson(level, supplier.get()); } @Override -- cgit v1.2.3 From 0d8c6fc3b5b9735918e20e0518ad680dc6cc82b4 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 21 Sep 2023 09:31:09 +0200 Subject: Add calculator for adjusting JVM heap size based on model size --- .../com/yahoo/config/model/api/OnnxModelCost.java | 30 +++++++ .../yahoo/vespa/model/DefaultOnnxModelCost.java | 99 ++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java new file mode 100644 index 00000000000..8c7f0db3bec --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.model.api; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; + +/** + * @author bjorncs + */ +public interface OnnxModelCost { + + Calculator newCalculator(DeployLogger logger); + + interface Calculator { + long aggregatedModelCostInBytes(); + void registerModel(ApplicationFile path); + void registerModel(ModelReference ref); + } + + static OnnxModelCost testInstance() { + return (__) -> new Calculator() { + @Override public long aggregatedModelCostInBytes() { return 0; } + @Override public void registerModel(ApplicationFile path) {} + @Override public void registerModel(ModelReference ref) {} + }; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java new file mode 100644 index 00000000000..76733872882 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java @@ -0,0 +1,99 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.OnnxModelCost; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; + +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * Aggregates estimated footprint of configured ONNX models. + * + * @author bjorncs + */ +public class DefaultOnnxModelCost implements OnnxModelCost { + + @Override + public Calculator newCalculator(DeployLogger logger) { + return new CalculatorImpl(logger); + } + + private static class CalculatorImpl implements Calculator { + private final DeployLogger log; + + private final ConcurrentMap modelCost = new ConcurrentHashMap<>(); + + private CalculatorImpl(DeployLogger log) { + this.log = log; + } + + @Override + public long aggregatedModelCostInBytes() { + return modelCost.values().stream().mapToLong(Long::longValue).sum(); + } + + @Override + public void registerModel(ApplicationFile f) { + String path = f.getPath().getRelative(); + if (alreadyAnalyzed(path)) return; + log.log(Level.FINE, () -> "Register model '%s'".formatted(path)); + deductJvmHeapSizeWithModelCost(f.exists() ? f.getSize() : 0, path); + } + + @Override + public void registerModel(ModelReference ref) { + log.log(Level.FINE, () -> "Register model '%s'".formatted(ref.toString())); + if (ref.path().isPresent()) { + var path = Paths.get(ref.path().get().value()); + var source = path.getFileName().toString(); + if (alreadyAnalyzed(source)) return; + deductJvmHeapSizeWithModelCost(uncheck(() -> Files.exists(path) ? Files.size(path) : 0), source); + } else if (ref.url().isPresent()) deductJvmHeapSizeWithModelCost(URI.create(ref.url().get().value())); + else throw new IllegalStateException(ref.toString()); + } + + private void deductJvmHeapSizeWithModelCost(URI uri) { + if (alreadyAnalyzed(uri.toString())) return; + if (uri.getScheme().equals("http") || uri.getScheme().equals("https")) { + try { + var timeout = Duration.ofSeconds(3); + var httpClient = HttpClient.newBuilder().connectTimeout(timeout).build(); + var request = HttpRequest.newBuilder(uri).timeout(timeout).method("HEAD", HttpRequest.BodyPublishers.noBody()).build(); + var response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); + var contentLength = response.headers().firstValue("Content-Length").orElse("0"); + log.log(Level.FINE, () -> "Got content length '%s' for '%s'".formatted(contentLength, uri)); + deductJvmHeapSizeWithModelCost(Long.parseLong(contentLength), uri.toString()); + } catch (IllegalArgumentException | InterruptedException | IOException e) { + log.log(Level.INFO, () -> "Failed to get model size for '%s': %s".formatted(uri, e.getMessage()), e); + } + } + } + + private void deductJvmHeapSizeWithModelCost(long size, String source) { + long fallbackModelSize = 1024*1024*1024; + long estimatedCost = Math.max(300*1024*1024, (long) (1.4D * (size > 0 ? size : fallbackModelSize) + 100*1024*1024)); + log.log(Level.FINE, () -> + "Estimated %s footprint for model of size %s ('%s')".formatted(mb(estimatedCost), mb(size), source)); + modelCost.put(source, estimatedCost); + } + + private boolean alreadyAnalyzed(String source) { return modelCost.containsKey(source); } + + private static String mb(long bytes) { return "%dMB".formatted(bytes / (1024*1024)); } + } +} -- cgit v1.2.3 From 5df57a04fff32458dd9aa5642d4b7f5f7919f614 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 22 Sep 2023 15:15:59 +0200 Subject: Aggregate Onnx model cost for JVM heap size calculation --- .../java/com/yahoo/config/model/api/OnnxModelCost.java | 3 +-- .../java/com/yahoo/config/model/deploy/DeployState.java | 14 ++++++++++++-- .../java/com/yahoo/vespa/model/VespaModelFactory.java | 4 +++- .../model/builder/xml/dom/DomComponentBuilder.java | 14 ++++++++------ .../model/container/ApplicationContainerCluster.java | 17 +++++++++++++++-- .../vespa/model/container/component/BertEmbedder.java | 4 +++- .../model/container/component/HuggingFaceEmbedder.java | 4 +++- .../vespa/model/container/search/ContainerSearch.java | 6 ++++++ .../model/container/xml/ContainerModelBuilder.java | 1 + 9 files changed, 52 insertions(+), 15 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index 8c7f0db3bec..422ceba8074 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -5,7 +5,6 @@ package com.yahoo.config.model.api; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.ApplicationId; /** * @author bjorncs @@ -20,7 +19,7 @@ public interface OnnxModelCost { void registerModel(ModelReference ref); } - static OnnxModelCost testInstance() { + static OnnxModelCost disabled() { return (__) -> new Calculator() { @Override public long aggregatedModelCostInBytes() { return 0; } @Override public void registerModel(ApplicationFile path) {} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 4df7a76031a..5ab258ecce8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.api.ValidationParameters; @@ -90,6 +91,7 @@ public class DeployState implements ConfigDefinitionStore { private final Provisioned provisioned; private final Reindexing reindexing; private final ExecutorService executor; + private final OnnxModelCost onnxModelCost; public static DeployState createTestState() { return new Builder().build(); @@ -124,7 +126,8 @@ public class DeployState implements ConfigDefinitionStore { boolean accessLoggingEnabledByDefault, Optional wantedDockerImageRepo, Reindexing reindexing, - Optional validationOverrides) { + Optional validationOverrides, + OnnxModelCost onnxModelCost) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.executor = executor; @@ -152,6 +155,7 @@ public class DeployState implements ConfigDefinitionStore { this.now = now; this.wantedDockerImageRepo = wantedDockerImageRepo; this.reindexing = reindexing; + this.onnxModelCost = onnxModelCost; } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { @@ -305,6 +309,8 @@ public class DeployState implements ConfigDefinitionStore { public Optional reindexing() { return Optional.ofNullable(reindexing); } + public OnnxModelCost onnxModelCost() { return onnxModelCost; } + public boolean isHostedTenantApplication(ApplicationType type) { boolean isTesterApplication = getProperties().applicationId().instance().isTester(); return isHosted() && type == ApplicationType.DEFAULT && !isTesterApplication; @@ -333,6 +339,7 @@ public class DeployState implements ConfigDefinitionStore { private QueryProfiles queryProfiles = null; private Reindexing reindexing = null; private Optional validationOverrides = Optional.empty(); + private OnnxModelCost onnxModelCost = OnnxModelCost.disabled(); public Builder() {} @@ -450,6 +457,8 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder onnxModelCost(OnnxModelCost instance) { this.onnxModelCost = instance; return this; } + public DeployState build() { return build(new ValidationParameters()); } @@ -482,7 +491,8 @@ public class DeployState implements ConfigDefinitionStore { accessLoggingEnabledByDefault, wantedDockerImageRepo, reindexing, - validationOverrides); + validationOverrides, + onnxModelCost); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 28ff8dff620..727a18aee2c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -21,6 +21,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; @@ -197,7 +198,8 @@ public class VespaModelFactory implements ModelFactory { .zone(zone) .now(clock.instant()) .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()) - .wantedDockerImageRepo(modelContext.wantedDockerImageRepo()); + .wantedDockerImageRepo(modelContext.wantedDockerImageRepo()) + .onnxModelCost(modelContext.properties().hostedVespa() ? new DefaultOnnxModelCost() : OnnxModelCost.disabled()); modelContext.previousModel().ifPresent(builder::previousModel); modelContext.reindexing().ifPresent(builder::reindexing); return builder.build(validationParameters); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index 7501f6162c7..ed89b810301 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -7,11 +7,12 @@ import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; -import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BertEmbedder; import com.yahoo.vespa.model.container.component.ColBertEmbedder; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.HuggingFaceEmbedder; +import com.yahoo.vespa.model.container.component.HuggingFaceTokenizer; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; @@ -35,19 +36,20 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde @Override protected Component, ?> doBuild(DeployState deployState, TreeConfigProducer ancestor, Element spec) { - var component = buildComponent(spec, deployState); + var component = buildComponent(spec, deployState, ancestor); addChildren(deployState, ancestor, spec, component); return component; } - private Component, ?> buildComponent(Element spec, DeployState state) { + private Component, ?> buildComponent( + Element spec, DeployState state, TreeConfigProducer ancestor) { if (spec.hasAttribute("type")) { var type = spec.getAttribute("type"); return switch (type) { - case "hugging-face-embedder" -> new HuggingFaceEmbedder(spec, state); + case "hugging-face-embedder" -> new HuggingFaceEmbedder((ApplicationContainerCluster)ancestor, spec, state); case "hugging-face-tokenizer" -> new HuggingFaceTokenizer(spec, state); - case "bert-embedder" -> new BertEmbedder(spec, state); case "colbert-embedder" -> new ColBertEmbedder(spec, state); + case "bert-embedder" -> new BertEmbedder((ApplicationContainerCluster)ancestor, spec, state); default -> throw new IllegalArgumentException("Unknown component type '%s'".formatted(type)); }; } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index b9021912244..49ad67f633f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -8,10 +8,12 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; @@ -47,6 +49,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; @@ -82,6 +85,8 @@ public final class ApplicationContainerCluster extends ContainerCluster applicationBundles = new LinkedHashSet<>(); private final Set previousHosts; + private final OnnxModelCost.Calculator onnxModelCost; + private final DeployLogger logger; private ContainerModelEvaluation modelEvaluation; @@ -125,6 +130,8 @@ public final class ApplicationContainerCluster extends ContainerCluster 0 ? Math.min(99, deployState.featureFlags().heapSizePercentage()) : defaultHeapSizePercentageOfAvailableMemory; + onnxModelCost = deployState.onnxModelCost().newCalculator(deployState.getDeployLogger()); + logger = deployState.getDeployLogger(); } @Override @@ -193,8 +200,12 @@ public final class ApplicationContainerCluster extends ContainerCluster "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" + .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); + return Optional.of(memoryPercentage); } return Optional.empty(); } @@ -373,6 +384,8 @@ public final class ApplicationContainerCluster extends ContainerCluster private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; + private ApplicationPackage app; public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); this.globalPhase = deployState.featureFlags().enableGlobalPhase(); this.useReconfigurableDispatcher = deployState.featureFlags().useReconfigurableDispatcher(); this.schemasWithGlobalPhase = getSchemasWithGlobalPhase(deployState); + this.app = deployState.getApplicationPackage(); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); @@ -96,6 +99,9 @@ public class ContainerSearch extends ContainerSubsystem if ( ! schemasWithGlobalPhase.contains(documentDb.getSchemaName())) continue; var factory = new RankProfilesEvaluatorComponent(documentDb); if ( ! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + var onnxModels = documentDb.getDerivedConfiguration().getRankProfileList().getOnnxModels(); + onnxModels.asMap().forEach( + (__, model) -> owningCluster.onnxModelCost().registerModel(app.getFile(model.getFilePath()))); owningCluster.addComponent(factory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 35b0213bf59..d9c4dea478c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -778,6 +778,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } + cluster.onnxModelCost().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath())); } cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); -- cgit v1.2.3 From 492556f0ebd216b01fc4af7e0d32e51f6ade6927 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 21 Sep 2023 11:21:46 +0200 Subject: Filter non-debug messages before asserting --- .../java/com/yahoo/config/model/provision/ModelProvisioningTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2f8a8bddf20..13f78029eef 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -288,10 +288,11 @@ public class ModelProvisioningTest { assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); - assertEquals(1, logger.msgs().size()); + var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); + assertEquals(1, msgs.size()); assertEquals("Declaring combined cluster with is deprecated without replacement, " + "and the feature will be removed in Vespa 9. Use separate container and content clusters instead", - logger.msgs().get(0).message); + msgs.get(0).message); } @Test -- cgit v1.2.3 From d73ec0dbe017f8e02067aabefa0f421d63e332db Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 21 Sep 2023 15:18:21 +0200 Subject: Add feature flag to enable new JVM heap size calculation --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 1 + .../main/java/com/yahoo/config/model/deploy/TestProperties.java | 4 ++++ .../yahoo/vespa/model/container/ApplicationContainerCluster.java | 4 +++- .../com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 +++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 +++++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 37b24f0ac1d..446c32801e0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -117,6 +117,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableNestedMultivalueGrouping() { return false; } @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } + @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 815c32e3c8f..77356292f9a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -86,6 +86,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean allowUserFilters = true; private List dataplaneTokens; private int contentLayerMetadataFeatureLevel = 0; + private boolean dynamicHeapSize = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -144,6 +145,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only @Override public List dataplaneTokens() { return dataplaneTokens; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -379,6 +381,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setDynamicHeapSize(boolean b) { this.dynamicHeapSize = b; return this; } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 49ad67f633f..eb62cc1f2a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -97,6 +97,7 @@ public final class ApplicationContainerCluster extends ContainerCluster parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); + dynamicHeapSize = deployState.featureFlags().dynamicHeapSize(); previousHosts = Collections.unmodifiableSet(deployState.getPreviousModel().stream() .map(Model::allocatedHosts) .map(AllocatedHosts::getHosts) @@ -200,7 +202,7 @@ public final class ApplicationContainerCluster extends ContainerCluster "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 142f98e13e3..3e33b345437 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext { private final boolean enableNestedMultivalueGrouping; private final boolean useReconfigurableDispatcher; private final int contentLayerMetadataFeatureLevel; + private final boolean dynamicHeapSize; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -243,6 +244,7 @@ public class ModelContextImpl implements ModelContext { this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING); this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER); this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL); + this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -293,6 +295,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; } @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } + @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } private static V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2e158f0f3ef..e5b76bedecd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -406,6 +406,13 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag DYNAMIC_HEAP_SIZE = defineFeatureFlag( + "dynamic-heap-size", false, + List.of("bjorncs"), "2023-09-21", "2024-01-15", + "Whether to calculate JVM heap size based on predicted Onnx model memory requirements", + "Takes effect at redeployment", + INSTANCE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List owners, String createdAt, String expiresAt, String description, -- cgit v1.2.3 From 603e792d986afec650b85f5ed47e78dc654bdb3d Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Thu, 21 Sep 2023 15:58:53 +0200 Subject: Set heapsize in addition to percentage Track nominal heap size as config for debugging purposes. --- .../model/container/ApplicationContainerCluster.java | 17 ++++++++++------- .../yahoo/vespa/model/container/ContainerCluster.java | 10 +++++++++- .../vespa/model/content/cluster/ContentCluster.java | 2 +- .../config/model/provision/ModelProvisioningTest.java | 2 +- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index eb62cc1f2a4..47fd85fde7d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -191,14 +191,14 @@ public final class ApplicationContainerCluster extends ContainerCluster getMemoryPercentage() { - if (memoryPercentage != null) return Optional.of(memoryPercentage); + public Optional getMemoryPercentage() { + if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { int availableMemoryPercentage = getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; - if (getContainers().isEmpty()) return Optional.of(availableMemoryPercentage); // Node memory is not known + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known // Node memory is known so convert available memory percentage to node memory percentage double totalMemory = getContainers().get(0).getHostResource().realResources().memoryGb(); @@ -207,7 +207,7 @@ public final class ApplicationContainerCluster extends ContainerCluster "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); - return Optional.of(memoryPercentage); + return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); } return Optional.empty(); } @@ -312,12 +312,15 @@ public final class ApplicationContainerCluster extends ContainerCluster builder.jvm.heapSizeAsPercentageOfPhysicalMemory(percentage)); + .minHeapsize(heapsize) + .heapsize(heapsize); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6bbc24e8739..fa13e7ec9d6 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -62,6 +62,7 @@ import com.yahoo.vespa.model.container.search.ContainerSearch; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; + import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; @@ -71,6 +72,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalDouble; import java.util.Set; import java.util.TreeSet; @@ -718,5 +720,11 @@ public abstract class ContainerCluster * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public Optional getMemoryPercentage() { return Optional.empty(); } + public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { + static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { + return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + } + } + public Optional getMemoryPercentage() { return Optional.empty(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bb72eda7d04..d18309ef0af 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -256,7 +256,7 @@ public class ContentCluster extends TreeConfigProducer implem for (ContainerModel containerModel : containers) { Optional hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; } } return 0.0; diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 13f78029eef..38f51323ee2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -148,7 +148,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(Optional.of(45), mydisc2.getMemoryPercentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); -- cgit v1.2.3 From eff9fc1b006dc526caa8499473337548305d3bf4 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 22 Sep 2023 15:12:02 +0200 Subject: Use node with smallest memory in heap size calculation --- .../com/yahoo/vespa/model/container/ApplicationContainerCluster.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 47fd85fde7d..d6403c2e8e3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -201,7 +201,9 @@ public final class ApplicationContainerCluster extends ContainerCluster c.getHostResource().realResources().memoryGb()).min().orElseThrow() + : getContainers().get(0).getHostResource().realResources().memoryGb(); double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCost.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); -- cgit v1.2.3 From d83fb9612cfef846273739c50fb6fdbd9c95de3a Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 22 Sep 2023 15:16:37 +0200 Subject: Add validator validating heap size calculation --- .../validation/JvmHeapSizeValidator.java | 51 +++++++++ .../model/application/validation/Validation.java | 1 + .../validation/JvmHeapSizeValidatorTest.java | 126 +++++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java create mode 100644 config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java new file mode 100644 index 00000000000..2c5e0db14b9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Level; + +/** + * Validates that the container node flavour has enough resources to run configured ONNX models. + * + * @author bjorncs + */ +public class JvmHeapSizeValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState ds) { + if (!ds.featureFlags().dynamicHeapSize()) return; + if (!ds.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + + model.getContainerClusters().forEach((clusterId, appCluster) -> { + var mp = appCluster.getMemoryPercentage().orElse(null); + if (mp == null) return; + if (mp.availableMemoryGb().isEmpty()) { + ds.getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); + return; + } + long jvmModelCost = appCluster.onnxModelCost().aggregatedModelCostInBytes(); + if (jvmModelCost > 0) { + int percentLimit = 10; + if (mp.percentage() < percentLimit) { + throw new IllegalArgumentException( + ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") + .formatted(clusterId, mp.percentage(), percentLimit, jvmModelCost / (1024D * 1024 * 1024))); + } + double gbLimit = 0.4; + double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + if (availableMemoryGb < gbLimit) { + throw new IllegalArgumentException( + ("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") + .formatted(clusterId, availableMemoryGb, gbLimit, jvmModelCost / (1024D * 1024 * 1024))); + } + } + }); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 53a553ee624..b9ecf7c2d22 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -87,6 +87,7 @@ public class Validation { new AccessControlFilterExcludeValidator().validate(model, deployState); new CloudUserFilterValidator().validate(model, deployState); new CloudHttpConnectorValidator().validate(model, deployState); + new JvmHeapSizeValidator().validate(model, deployState); additionalValidators.forEach(v -> v.validate(model, deployState)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java new file mode 100644 index 00000000000..086f2fe778f --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -0,0 +1,126 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author bjorncs + */ +class JvmHeapSizeValidatorTest { + + @Test + void fails_on_too_low_jvm_percentage() throws IOException, SAXException { + var deployState = createDeployState(8, 7L * 1024 * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 10%). Estimated cost of ONNX models is 7.00GB"; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void fails_on_too_low_heap_size() throws IOException, SAXException { + var deployState = createDeployState(2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); + String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.30GB < 0.40GB). Estimated cost of ONNX models is 1.00GB."; + assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + } + + @Test + void accepts_adequate_heap_size() throws IOException, SAXException { + var deployState = createDeployState(8, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + @Test + void accepts_services_with_explicit_jvm_size() throws IOException, SAXException { + String servicesXml = + """ + + + + + + + + + + + + + """; + var deployState = createDeployState(servicesXml, 2, 1024L * 1024 * 1024); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + assertDoesNotThrow(() -> new JvmHeapSizeValidator().validate(model, deployState)); + } + + private static DeployState createDeployState(String servicesXml, double nodeGb, long modelCostBytes) { + return new DeployState.Builder() + .applicationPackage( + new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build()) + .modelHostProvisioner(new InMemoryProvisioner(5, new NodeResources(4, nodeGb, 125, 0.3), true)) + .properties(new TestProperties().setHostedVespa(true).setDynamicHeapSize(true)) + .onnxModelCost(new ModelCostDummy(modelCostBytes)) + .build(); + } + + private static DeployState createDeployState(double nodeGb, long modelCostBytes) { + String servicesXml = + """ + + + + + + + + + + + + """.formatted(nodeGb); + return createDeployState(servicesXml, nodeGb, modelCostBytes); + } + + private static class ModelCostDummy implements OnnxModelCost, OnnxModelCost.Calculator { + final AtomicLong totalCost = new AtomicLong(); + final long modelCost; + + ModelCostDummy(long modelCost) { this.modelCost = modelCost; } + + @Override public Calculator newCalculator(DeployLogger logger) { return this; } + @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } + @Override public void registerModel(ApplicationFile path) {} + + @Override + public void registerModel(ModelReference ref) { + assertEquals("https://my/url/model.onnx", ref.url().orElseThrow().value().toString()); + totalCost.addAndGet(modelCost); + } + } + +} \ No newline at end of file -- cgit v1.2.3 From d261ac6f01603eaf545040e3eb45967ca78d933c Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 22 Sep 2023 15:30:36 +0200 Subject: Aggregate Onnx model cost for Colbert embedder --- .../com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java | 2 +- .../com/yahoo/vespa/model/container/component/ColBertEmbedder.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index ed89b810301..9ecd359f90d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -48,7 +48,7 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde return switch (type) { case "hugging-face-embedder" -> new HuggingFaceEmbedder((ApplicationContainerCluster)ancestor, spec, state); case "hugging-face-tokenizer" -> new HuggingFaceTokenizer(spec, state); - case "colbert-embedder" -> new ColBertEmbedder(spec, state); + case "colbert-embedder" -> new ColBertEmbedder((ApplicationContainerCluster)ancestor, spec, state); case "bert-embedder" -> new BertEmbedder((ApplicationContainerCluster)ancestor, spec, state); default -> throw new IllegalArgumentException("Unknown component type '%s'".formatted(type)); }; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java index c0fdfe3dc64..63096ebcbe2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.ColBertEmbedderConfig; -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.xml.ModelIdResolver; import org.w3c.dom.Element; @@ -40,7 +40,7 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo private final Integer onnxIntraopThreads; private final Integer onnxGpuDevice; - public ColBertEmbedder(Element xml, DeployState state) { + public ColBertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.ColBertEmbedder", INTEGRATION_BUNDLE_NAME, xml); var transformerModelElem = getOptionalChild(xml, "transformer-model").orElseThrow(); model = ModelIdResolver.resolveToModelReference(transformerModelElem, state); @@ -60,7 +60,7 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); - + cluster.onnxModelCost().registerModel(model); } private static ModelReference resolveDefaultVocab(Element model, DeployState state) { -- cgit v1.2.3 From 99c901c271242ff73eb16e35bbb499d3a96dad1e Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Fri, 22 Sep 2023 15:35:35 +0200 Subject: Use `Curator.getSize()` to get size for path --- .../java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java | 6 ++++++ .../com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 4c262379c35..1288b63cadd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -111,6 +111,12 @@ public class ZKApplication { return getBytesInternal(getFullPath(path)); } + public long getSize(Path path) { + return curator.getStat(path).map(stat -> (long)stat.getDataLength()) + .orElseThrow(() -> new IllegalArgumentException( + "Could not get size from '" + path + "' in zookeeper")); + } + void putData(Path path, String data) { byte[] bytes = Utf8.toBytes(data); ensureDataIsNotTooLarge(bytes, path); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 13bf4b2368b..e51f8627de2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -183,7 +183,7 @@ class ZKApplicationFile extends ApplicationFile { } } - @Override public long getSize() { return zkApp.getBytes(getZKPath(path)).length; } + @Override public long getSize() { return zkApp.getSize(getZKPath(path)); } @Override public int compareTo(ApplicationFile other) { -- cgit v1.2.3 From 242359e81f1740b53f44422f90bc36a21ec516e2 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 22 Sep 2023 16:51:35 +0200 Subject: Add test --- .../prelude/semantics/test/EquivTestCase.java | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java new file mode 100644 index 00000000000..3e2c634b7f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/EquivTestCase.java @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.jupiter.api.Test; + +/** + * @author bratseth + */ +public class EquivTestCase extends RuleBaseAbstractTestCase { + + public EquivTestCase() { + super("equiv.sr"); + } + + @Test + void testEquiv() { + assertSemantics("EQUIV \"lord of the rings\" lotr", "lotr"); + } + + @Test + void testEquivWithFollowingQuery() { + assertSemantics("AND (EQUIV \"lord of the rings\" lotr) is a movie", "lotr is a movie"); + } + + @Test + void testEquivWithPrecedingQuery() { + assertSemantics("AND a movie is (EQUIV \"lord of the rings\" lotr)", "a movie is lotr"); + } + + @Test + void testEquivWithSurroundingQuery() { + assertSemantics("AND a movie is (EQUIV \"lord of the rings\" lotr) yes", "a movie is lotr yes"); + } + +} -- cgit v1.2.3 From 82730b650885f5a1f366980ff7cdf900aa0844ff Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 22 Sep 2023 16:53:50 +0200 Subject: Output gram for summary fields where the source is gram --- .../java/com/yahoo/schema/derived/IndexInfo.java | 9 ++++++- .../yahoo/vespa/documentmodel/SummaryField.java | 28 +++++++++------------- config-model/src/test/derived/ngram/chunk.sd | 20 ++++++++++++++++ config-model/src/test/derived/ngram/index-info.cfg | 21 ++++++++++++++++ .../com/yahoo/schema/derived/NGramTestCase.java | 15 ++++++++++++ .../yahoo/search/querytransform/NGramSearcher.java | 4 ++-- .../querytransform/test/NGramSearcherTestCase.java | 8 +++---- 7 files changed, 81 insertions(+), 24 deletions(-) create mode 100644 config-model/src/test/derived/ngram/chunk.sd create mode 100644 config-model/src/test/derived/ngram/index-info.cfg create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 7d88985b2d5..a21aefc9f71 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -82,7 +82,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } // Commands for summary fields - // TODO: Move to fieldinfo and implement differently. This is not right + // TODO: Move to schemainfo and implement differently for (SummaryField summaryField : schema.getUniqueNamedSummaryFields().values()) { if (summaryField.getTransform().isTeaser()) { addIndexCommand(summaryField.getName(), CMD_DYNTEASER); @@ -90,6 +90,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (summaryField.getTransform().isBolded()) { addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); } + + var sourceField = schema.getField(summaryField.getSourceField()); // Take the first as they should all be consistent + if (sourceField != null && sourceField.getMatching().getType().equals(MatchType.GRAM)) { + addIndexCommand(summaryField.getName(), + "ngram " + (sourceField.getMatching().getGramSize() > 0 ? sourceField.getMatching().getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE)); + + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 7439e65dee6..49cd36e4bc2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -17,9 +17,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public class SummaryField extends Field implements Cloneable, TypedKey { - /** - * A source (field name). - */ + /** A source (field name). */ public static class Source implements Serializable { private final String name; @@ -38,12 +36,8 @@ public class SummaryField extends Field implements Cloneable, TypedKey { @Override public boolean equals(Object obj) { - if (!(obj instanceof Source)) { - return false; - } - Source other = (Source)obj; - return name.equals(other.name) && - override == other.override; + if (!(obj instanceof Source other)) return false; + return name.equals(other.name) && override == other.override; } @Override @@ -67,14 +61,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ private Set sources = new java.util.LinkedHashSet<>(); - private Set destinations=new java.util.LinkedHashSet<>(); + private Set destinations =new java.util.LinkedHashSet<>(); /** True if this field was defined implicitly */ - private boolean implicit=false; + private boolean implicit = false; /** Creates a summary field with NONE as transform */ public SummaryField(String name, DataType type) { - this(name,type, SummaryTransform.NONE); + this(name, type, SummaryTransform.NONE); } /** Creates a summary field with NONE as transform */ @@ -97,7 +91,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public boolean isImplicit() { return implicit; } public void setTransform(SummaryTransform transform) { - this.transform=transform; + this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { // This is the kind of logic we want to have in processing, // but can't because of deriveDocuments mode, which doesn't run @@ -110,9 +104,9 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns the first source field of this, or null if the source field is not present */ public String getSourceField() { - String sourceName=getName(); - if (sources.size()>0) - sourceName=sources.iterator().next().getName(); + String sourceName = getName(); + if ( ! sources.isEmpty()) + sourceName = sources.iterator().next().getName(); return sourceName; } @@ -137,7 +131,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns the first source name of this, or the field name if no source has been set */ public String getSingleSource() { - if (sources.size()==0) return getName(); + if (sources.isEmpty()) return getName(); return sources.iterator().next().getName(); } diff --git a/config-model/src/test/derived/ngram/chunk.sd b/config-model/src/test/derived/ngram/chunk.sd new file mode 100644 index 00000000000..7c2a7465327 --- /dev/null +++ b/config-model/src/test/derived/ngram/chunk.sd @@ -0,0 +1,20 @@ +schema chunk { + + document chunk { + field content type string { + indexing: summary | index + match { + gram + gram-size: 3 + } + } + } + + document-summary content-summary inherits default { + summary content_dynamic type string { + source: content + dynamic + } + } + +} diff --git a/config-model/src/test/derived/ngram/index-info.cfg b/config-model/src/test/derived/ngram/index-info.cfg new file mode 100644 index 00000000000..72b6760ceb5 --- /dev/null +++ b/config-model/src/test/derived/ngram/index-info.cfg @@ -0,0 +1,21 @@ +indexinfo[].name "chunk" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "ngram 3" +indexinfo[].command[].indexname "content_dynamic" +indexinfo[].command[].command "dynteaser" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java new file mode 100644 index 00000000000..4481445858a --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/NGramTestCase.java @@ -0,0 +1,15 @@ +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +public class NGramTestCase extends AbstractExportingTestCase { + + @Test + void testNGram() throws IOException, ParseException { + assertCorrectDeriving("ngram"); + } + +} \ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 3edad64f9f2..3dfa278662d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -135,14 +135,14 @@ public class NGramSearcher extends Searcher { * Creates the root of the query subtree which will contain the grams to match, * called by {@link #splitToGrams}. This hook is provided to make it easy to create a subclass which * matches grams using a different composite item, e.g an OrItem. - *

+ * * This default implementation returns createGramRoot(query). * * @param term the term item this gram root is replacing in the query tree, * typically used to access the index name of the term when that is required by the new gram root * (such as in PhraseItem) * @param query the input query, to make it possible to return a different composite item type - * depending on the query content + * depending on the query content * @return the composite item to add the gram items to in {@link #splitToGrams} */ protected CompositeItem createGramRoot(HasIndexItem term, Query query) { diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java index 49449153d1f..4d25ebdccff 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -334,15 +334,15 @@ public class NGramSearcherTestCase { Result r = new Execution(new Chain<>(createSearcher(), new MockBackend1()), createContextStub(createIndexFacts())).search(q); Hit h1 = r.hits().get("hit1"); assertEquals("Should be untouched,\u001feven if containing \u001f", - h1.getField("test").toString()); + h1.getField("test").toString()); assertTrue(h1.getField("test") instanceof String); assertEquals("Blue red Ed A", h1.getField("gram2").toString()); assertTrue(h1.getField("gram2") instanceof XMLString); assertEquals("Blue red ed a\u001f", - h1.getField("gram3").toString(), - "Separators on borders work"); + h1.getField("gram3").toString(), + "Separators on borders work"); assertTrue(h1.getField("gram3") instanceof String); Hit h2 = r.hits().get("hit2"); @@ -352,7 +352,7 @@ public class NGramSearcherTestCase { Hit h3 = r.hits().get("hit3"); assertEquals("\u001ffin\u001f \u001fen\u001f \u001fa\u001f", h3.getField("gram2").toString()); assertEquals("#Logging in #Java is like that \"Judean P\u001fopul\u001far Front\" scene from \"Life of Brian\".", - h3.getField("gram3").toString()); + h3.getField("gram3").toString()); } private Item parseQuery(String query, Query.Type type) { -- cgit v1.2.3 From 297b712e37a889b824f5b1ef241379d67d045ecd Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 22 Sep 2023 17:04:50 +0200 Subject: Use OptionalInt --- .../main/java/com/yahoo/schema/derived/IndexInfo.java | 4 ++-- .../main/java/com/yahoo/schema/document/Matching.java | 18 +++++++++++------- .../java/com/yahoo/schema/processing/NGramMatch.java | 6 ++---- .../search/IndexingScriptChangeMessageBuilder.java | 3 ++- .../com/yahoo/schema/processing/NGramTestCase.java | 6 +++--- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index a21aefc9f71..f6a022e9930 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -94,7 +94,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { var sourceField = schema.getField(summaryField.getSourceField()); // Take the first as they should all be consistent if (sourceField != null && sourceField.getMatching().getType().equals(MatchType.GRAM)) { addIndexCommand(summaryField.getName(), - "ngram " + (sourceField.getMatching().getGramSize() > 0 ? sourceField.getMatching().getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE)); + "ngram " + (sourceField.getMatching().getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE))); } } @@ -459,7 +459,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) - .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); + .command("ngram " + fieldSetMatching.getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE))); } else if (fieldSetMatching.getType().equals(MatchType.TEXT)) { } diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 1fe947d672b..264fd0ff3b9 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -1,7 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.document; +import com.yahoo.schema.processing.NGramMatch; + import java.io.Serializable; +import java.util.OptionalInt; /** * Defines how a field should be matched. @@ -23,8 +26,8 @@ public class Matching implements Cloneable, Serializable { private boolean algorithmUserSet = false; - /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ - private int gramSize = -1; + /** The gram size is the n in n-gram, or empty if not set. Should only be set with gram matching. */ + private OptionalInt gramSize = OptionalInt.empty(); /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ private Integer maxLength; @@ -67,10 +70,10 @@ public class Matching implements Cloneable, Serializable { public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } - /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ - public int getGramSize() { return gramSize; } + /** Returns the gram size, or empty if not set. Should only be set with gram matching. */ + public OptionalInt getGramSize() { return gramSize; } - public void setGramSize(int gramSize) { this.gramSize=gramSize; } + public void setGramSize(int gramSize) { this.gramSize = OptionalInt.of(gramSize); } /** * Merge data from another matching object @@ -107,10 +110,11 @@ public class Matching implements Cloneable, Serializable { @Override public String toString() { - return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + - "], [exact-terminator "+exactMatchTerminator+"]"; + return type + " matching [" + (type == MatchType.GRAM ? "gram size " + gramSize.orElse(NGramMatch.DEFAULT_GRAM_SIZE) : "supports " + algorithm) + + "], [exact-terminator " + exactMatchTerminator + "]"; } + @Override public Matching clone() { try { return (Matching)super.clone(); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index f1ff910be43..6ec5428156f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -31,7 +31,7 @@ public class NGramMatch extends Processor { for (SDField field : schema.allConcreteFields()) { if (field.getMatching().getType().equals(MatchType.GRAM)) implementGramMatch(schema, field, validate); - else if (validate && field.getMatching().getGramSize() >= 0) + else if (validate && field.getMatching().getGramSize().isPresent()) throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); } } @@ -40,9 +40,7 @@ public class NGramMatch extends Processor { if (validate && field.doesAttributing() && ! field.doesIndexing()) throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' in indexing"); - int n = field.getMatching().getGramSize(); - if (n < 0) - n = DEFAULT_GRAM_SIZE; // not set - use default gram size + int n = field.getMatching().getGramSize().orElse(DEFAULT_GRAM_SIZE); if (validate && n == 0) throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); field.getNormalizing().inferCodepoint(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index bbfa939f8a3..f265f2d09a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.NormalizeLevel; import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -89,7 +90,7 @@ public class IndexingScriptChangeMessageBuilder { MatchType type = matching.getType(); String retval = type.getName(); if (type == MatchType.GRAM) { - retval += " (size " + matching.getGramSize() + ")"; + retval += " (size " + matching.getGramSize().orElse(NGramMatch.DEFAULT_GRAM_SIZE) + ")"; } return retval; } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java index 06ea202b9c3..551542a84ba 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -27,15 +27,15 @@ public class NGramTestCase extends AbstractSchemaTestCase { SDField gram1 = schema.getConcreteField("gram_1"); assertEquals(MatchType.GRAM, gram1.getMatching().getType()); - assertEquals(1, gram1.getMatching().getGramSize()); + assertEquals(1, gram1.getMatching().getGramSize().getAsInt()); SDField gram2 = schema.getConcreteField("gram_2"); assertEquals(MatchType.GRAM, gram2.getMatching().getType()); - assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly + assertTrue(gram2.getMatching().getGramSize().isEmpty()); SDField gram3 = schema.getConcreteField("gram_3"); assertEquals(MatchType.GRAM, gram3.getMatching().getType()); - assertEquals(3, gram3.getMatching().getGramSize()); + assertEquals(3, gram3.getMatching().getGramSize().getAsInt()); assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); assertEquals("input gram_2 | ngram 2 | attribute gram_2 | index gram_2", gram2.getIndexingScript().iterator().next().toString()); -- cgit v1.2.3 From 60eb4d43fce41b252f9e16cf951a009f1d64f0ed Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Fri, 22 Sep 2023 17:27:51 +0200 Subject: Add prefix_size constructor argument to DfaFuzzyMatcher. --- .../dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp | 61 ++++++++++++++++++---- .../searchlib/attribute/dfa_fuzzy_matcher.cpp | 59 +++++++++++++++++++-- .../vespa/searchlib/attribute/dfa_fuzzy_matcher.h | 29 +++++++--- vespalib/src/vespa/vespalib/text/utf8.h | 1 + 4 files changed, 128 insertions(+), 22 deletions(-) diff --git a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp index c7bd0e917f3..77e23a58163 100644 --- a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp +++ b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp @@ -26,6 +26,7 @@ using namespace search::attribute; using namespace search; using vespalib::FuzzyMatcher; using vespalib::datastore::AtomicEntryRef; +using vespalib::datastore::EntryRef; using vespalib::fuzzy::LevenshteinDfa; using StringEnumStore = EnumStoreT; @@ -109,11 +110,11 @@ struct MatchStats { template void -brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, MatchStats& stats, StringVector& matched_words) +brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - FuzzyMatcher matcher(target, 2, 0, false); + FuzzyMatcher matcher(target, 2, prefix_size, false); auto itr = view.begin(); size_t matches = 0; size_t seeks = 0; @@ -133,15 +134,27 @@ brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumS template void -dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, MatchStats& stats, StringVector& matched_words) +dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - DfaFuzzyMatcher matcher(target, 2, false, LevenshteinDfa::DfaType::Explicit); - auto itr = view.begin(); + DfaFuzzyMatcher matcher(target, 2, prefix_size, false, LevenshteinDfa::DfaType::Explicit); + std::string target_copy(target.substr(0, prefix_size)); + auto prefix_cmp = store.make_folded_comparator_prefix(target_copy.c_str()); + auto itr = prefix_size > 0 ? view.lowerBound(AtomicEntryRef(), prefix_cmp) : view.begin(); + auto itr_end = itr; + if (itr_end.valid()) { + if (prefix_size > 0) { + if (!prefix_cmp.less(EntryRef(), itr_end.getKey().load_relaxed())) { + itr_end.seekPast(AtomicEntryRef(), prefix_cmp); + } + } else { + itr_end.end(); + } + } size_t matches = 0; size_t seeks = 0; - while (itr.valid()) { + while (itr != itr_end) { auto word = store.get_value(itr.getKey().load_relaxed()); if (matcher.is_match(word, itr, store.get_data_store())) { ++itr; @@ -170,15 +183,19 @@ struct DfaFuzzyMatcherTest : public ::testing::Test { updater.commit(); store.freeze_dictionary(); } - void expect_matches(std::string_view target, const StringVector& exp_matches) { + void expect_prefix_matches(std::string_view target, uint32_t prefix_size, const StringVector& exp_matches) { MatchStats stats; StringVector brute_force_matches; StringVector dfa_matches; - brute_force_fuzzy_match_in_dictionary(target, store, stats, brute_force_matches); - dfa_fuzzy_match_in_dictionary(target, store, stats, dfa_matches); + SCOPED_TRACE(target); + brute_force_fuzzy_match_in_dictionary(target, store, prefix_size, stats, brute_force_matches); + dfa_fuzzy_match_in_dictionary(target, store, prefix_size, stats, dfa_matches); EXPECT_EQ(exp_matches, brute_force_matches); EXPECT_EQ(exp_matches, dfa_matches); } + void expect_matches(std::string_view target, const StringVector& exp_matches) { + expect_prefix_matches(target, 0, exp_matches); + } }; TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) @@ -194,6 +211,28 @@ TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) expect_matches("forcecast", {"forecast"}); } +TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) +{ + StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", + "for", "forbid", "force", "ford", "forearm", "forecast", "forest" }; + populate_dictionary(words); + expect_prefix_matches("a", 1, {}); + expect_prefix_matches("b", 1, {"bob"}); + expect_prefix_matches("board", 1, {"board", "boat"}); + expect_prefix_matches("c", 1, {}); + expect_prefix_matches("food", 1, {"food", "foot", "for", "ford"}); + expect_prefix_matches("food", 2, {"food", "foot", "for", "ford"}); + expect_prefix_matches("food", 3, {"food", "foot"}); + expect_prefix_matches("foothill", 1, {"football", "foothill"}); + expect_prefix_matches("for", 1, {"food", "foot", "for", "force", "ford"}); + expect_prefix_matches("for", 2, {"food", "foot", "for", "force", "ford"}); + expect_prefix_matches("for", 3, {"for", "force", "ford"}); + expect_prefix_matches("force", 1, {"for", "force", "ford"}); + expect_prefix_matches("forcecast", 1, {"forecast"}); + expect_prefix_matches("forcecast", 4, {}); + expect_prefix_matches("z", 1, {}); +} + void benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool dfa_algorithm) { @@ -202,9 +241,9 @@ benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDicti for (size_t i = 0; i < std::min(words_to_match, dict.size()); ++i) { const auto& entry = dict[i]; if (dfa_algorithm) { - dfa_fuzzy_match_in_dictionary(entry.first, store, stats, dummy); + dfa_fuzzy_match_in_dictionary(entry.first, store, 0, stats, dummy); } else { - brute_force_fuzzy_match_in_dictionary(entry.first, store, stats, dummy); + brute_force_fuzzy_match_in_dictionary(entry.first, store, 0, stats, dummy); } } std::cout << (dfa_algorithm ? "DFA:" : "Brute force:") << " samples=" << stats.samples << ", avg_matches=" << stats.avg_matches() << ", avg_seeks=" << stats.avg_seeks() << ", avg_elapsed_ms=" << stats.avg_elapsed_ms() << std::endl; diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp index 580c34bd5d0..f66dd56c72f 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp @@ -1,17 +1,70 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dfa_fuzzy_matcher.h" +#include +#include using vespalib::fuzzy::LevenshteinDfa; +using vespalib::LowerCase; +using vespalib::Utf8Reader; +using vespalib::Utf8ReaderForZTS; namespace search::attribute { -DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, bool cased, LevenshteinDfa::DfaType dfa_type) - : _dfa(vespalib::fuzzy::LevenshteinDfa::build(target, max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), - _successor() +namespace { + +std::vector +extract_prefix(std::string_view target, uint32_t prefix_size, bool cased) +{ + std::vector result; + result.reserve(prefix_size); + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + uint32_t code_point = reader.getChar(); + if (!cased) { + code_point = LowerCase::convert(code_point); + } + result.emplace_back(code_point); + } + return result; +} + +std::string_view +extract_suffix(std::string_view target, uint32_t prefix_size) +{ + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + (void) reader.getChar(); + } + std::string_view result = target; + result.remove_prefix(reader.getPos()); + return result; +} + +} + +DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, LevenshteinDfa::DfaType dfa_type) + : _dfa(vespalib::fuzzy::LevenshteinDfa::build(extract_suffix(target, prefix_size), max_edits, (cased ? LevenshteinDfa::Casing::Cased : LevenshteinDfa::Casing::Uncased), dfa_type)), + _successor(), + _prefix(extract_prefix(target, prefix_size, cased)), + _successor_suffix(), + _prefix_size(prefix_size) { + _successor = _prefix; } DfaFuzzyMatcher::~DfaFuzzyMatcher() = default; +const char* +DfaFuzzyMatcher::skip_prefix(const char* word) const +{ + Utf8ReaderForZTS reader(word); + size_t pos = 0; + for (; pos < _prefix_size && reader.hasMore(); ++pos) { + (void) reader.getChar(); + } + assert(pos == _prefix_size); + return reader.get_current_ptr(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h index fcba13f85a4..3d24948e044 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h @@ -17,22 +17,35 @@ namespace search::attribute { class DfaFuzzyMatcher { private: vespalib::fuzzy::LevenshteinDfa _dfa; - std::vector _successor; + std::vector _successor; + std::vector _prefix; + std::vector _successor_suffix; + uint32_t _prefix_size; + const char*skip_prefix(const char* word) const; public: - DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); + DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); ~DfaFuzzyMatcher(); template bool is_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) { - auto match = _dfa.match(word, _successor); - if (match.matches()) { - return true; + if (_prefix_size > 0) { + word = skip_prefix(word); + auto match = _dfa.match(word, _successor_suffix); + if (match.matches()) { + return true; + } + _successor.resize(_prefix.size()); + _successor.insert(_successor.end(), _successor_suffix.begin(), _successor_suffix.end()); } else { - DfaStringComparator cmp(data_store, _successor); - itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); - return false; + auto match = _dfa.match(word, _successor); + if (match.matches()) { + return true; + } } + DfaStringComparator cmp(data_store, _successor); + itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); + return false; } }; diff --git a/vespalib/src/vespa/vespalib/text/utf8.h b/vespalib/src/vespa/vespalib/text/utf8.h index 99e3f8cfe13..489b16b1ed4 100644 --- a/vespalib/src/vespa/vespalib/text/utf8.h +++ b/vespalib/src/vespa/vespalib/text/utf8.h @@ -321,6 +321,7 @@ public: return i; } + const char* get_current_ptr() const noexcept { return _p; } }; -- cgit v1.2.3 From 0f5fddc01f36a7477a1db04a4c3752f003b2a10c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 21 Sep 2023 19:10:28 +0000 Subject: Wire in doom and let hitrate be a float int --- .../src/tests/proton/matching/query_test.cpp | 6 ++-- .../proton/matching/attribute_limiter.cpp | 3 +- .../searchcore/proton/matching/match_tools.cpp | 4 +-- .../src/vespa/searchcore/proton/matching/query.cpp | 9 ++--- .../src/vespa/searchcore/proton/matching/query.h | 5 +-- .../blueprint/intermediate_blueprints_test.cpp | 6 ++-- .../src/vespa/searchlib/queryeval/blueprint.cpp | 2 +- .../searchlib/queryeval/dot_product_blueprint.cpp | 2 +- .../src/vespa/searchlib/queryeval/executeinfo.cpp | 14 ++------ .../src/vespa/searchlib/queryeval/executeinfo.h | 38 ++++++++++++++++------ .../searchlib/queryeval/same_element_blueprint.cpp | 2 +- .../queryeval/wand/parallel_weak_and_blueprint.cpp | 15 ++++----- .../queryeval/weighted_set_term_blueprint.cpp | 8 ++--- 13 files changed, 62 insertions(+), 52 deletions(-) diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 575a52d01fb..d098bdde8b6 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -711,7 +711,7 @@ void Test::requireThatQueryGluesEverythingTogether() { EXPECT_EQUAL(1u, md->getNumTermFields()); query.optimize(); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); } @@ -744,7 +744,7 @@ void checkQueryAddsLocation(const string &loc_in, const string &loc_out) { MatchData::UP md = mdl.createMatchData(); EXPECT_EQUAL(2u, md->getNumTermFields()); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); if (!EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_out))) { @@ -966,7 +966,7 @@ Test::requireThatWhiteListBlueprintCanBeUsed() MatchData::UP md = mdl.createMatchData(); query.optimize(); - query.fetchPostings(); + query.fetchPostings(requestContext.getDoom()); SearchIterator::UP search = query.createSearch(*md); SimpleResult exp = SimpleResult().addHit(1).addHit(5).addHit(7).addHit(11); SimpleResult act; diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp index 1528b327747..b8027bff04a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -98,7 +99,7 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool st FieldSpecList field; // single field API is protected field.add(FieldSpec(_attribute_name, my_field_id, my_handle)); _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node); - _blueprint->fetchPostings(ExecuteInfo::create(strictSearch)); + _blueprint->fetchPostings(ExecuteInfo::create(strictSearch, &_requestContext.getDoom())); _estimatedHits.store(_blueprint->getState().estimate().estHits, std::memory_order_relaxed); _blueprint->freeze(); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 5ae671b88cb..758ef35ebc9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -201,9 +201,9 @@ MatchToolsFactory(QueryLimiter & queryLimiter, trace.addEvent(5, "Optimize query execution plan"); _query.optimize(); trace.addEvent(4, "Perform dictionary lookups and posting lists initialization"); - _query.fetchPostings(); + _query.fetchPostings(_requestContext.getDoom()); if (is_search) { - _query.handle_global_filter(searchContext.getDocIdLimit(), + _query.handle_global_filter(_requestContext.getDoom(), searchContext.getDocIdLimit(), _attribute_blueprint_params.global_filter_lower_limit, _attribute_blueprint_params.global_filter_upper_limit, thread_bundle, trace); diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index d0738f1857f..22f6ec9cc88 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -247,13 +247,14 @@ Query::optimize() } void -Query::fetchPostings() +Query::fetchPostings(const vespalib::Doom & doom) { - _blueprint->fetchPostings(search::queryeval::ExecuteInfo::create(true, 1.0)); + _blueprint->fetchPostings(search::queryeval::ExecuteInfo::create(true, &doom)); } void -Query::handle_global_filter(uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, +Query::handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, + double global_filter_lower_limit, double global_filter_upper_limit, vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace) { if (!handle_global_filter(*_blueprint, docid_limit, global_filter_lower_limit, global_filter_upper_limit, thread_bundle, &trace)) { @@ -264,7 +265,7 @@ Query::handle_global_filter(uint32_t docid_limit, double global_filter_lower_lim _blueprint = Blueprint::optimize(std::move(_blueprint)); LOG(debug, "blueprint after handle_global_filter:\n%s\n", _blueprint->asString().c_str()); // strictness may change if optimized order changed: - fetchPostings(); + fetchPostings(doom); } bool diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index b0299307e92..1a3136042a7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -98,9 +98,10 @@ public: * test to verify the original query without optimization. **/ void optimize(); - void fetchPostings(); + void fetchPostings(const vespalib::Doom & doom); - void handle_global_filter(uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, + void handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, + double global_filter_lower_limit, double global_filter_upper_limit, vespalib::ThreadBundle &thread_bundle, search::engine::Trace& trace); /** diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index ef0fd56840a..c617db871a7 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -128,9 +128,9 @@ TEST("test And propagates updated histestimate") { const RememberExecuteInfo & child = dynamic_cast(bp.getChild(i)); EXPECT_EQUAL((i == 0), child.executeInfo.isStrict()); } - EXPECT_EQUAL(1.0, dynamic_cast(bp.getChild(0)).executeInfo.hitRate()); - EXPECT_EQUAL(1.0/250, dynamic_cast(bp.getChild(1)).executeInfo.hitRate()); - EXPECT_EQUAL(1.0/(250*25), dynamic_cast(bp.getChild(2)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f, dynamic_cast(bp.getChild(0)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f/250, dynamic_cast(bp.getChild(1)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f/(250*25), dynamic_cast(bp.getChild(2)).executeInfo.hitRate()); } TEST("test And Blueprint") { diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 3f6085ef7ff..94d1a4917fd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -621,7 +621,7 @@ IntermediateBlueprint::fetchPostings(const ExecuteInfo &execInfo) double nextHitRate = execInfo.hitRate(); for (size_t i = 0; i < _children.size(); ++i) { Blueprint & child = *_children[i]; - child.fetchPostings(ExecuteInfo::create(execInfo.isStrict() && inheritStrict(i), nextHitRate)); + child.fetchPostings(ExecuteInfo::create(execInfo.isStrict() && inheritStrict(i), nextHitRate, execInfo.getDoom())); nextHitRate = computeNextHitRate(child, nextHitRate); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 795f5f1424a..4322cafb5c8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -66,7 +66,7 @@ DotProductBlueprint::createFilterSearch(bool strict, FilterConstraint constraint void DotProductBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); for (size_t i = 0; i < _terms.size(); ++i) { _terms[i]->fetchPostings(childInfo); } diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp index 604e20d2262..e5d20f047f5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.cpp @@ -4,17 +4,7 @@ namespace search::queryeval { -const ExecuteInfo ExecuteInfo::TRUE(true, 1.0); -const ExecuteInfo ExecuteInfo::FALSE(false, 1.0); - -ExecuteInfo -ExecuteInfo::create(bool strict) { - return create(strict, 1.0); -} - -ExecuteInfo -ExecuteInfo::create(bool strict, double hitRate) { - return ExecuteInfo(strict, hitRate); -} +const ExecuteInfo ExecuteInfo::TRUE(true, 1.0, nullptr); +const ExecuteInfo ExecuteInfo::FALSE(false, 1.0, nullptr); } diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h index e161b2bdab7..2dd34284bef 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2019 Oath inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include + namespace search::queryeval { /** @@ -11,20 +12,37 @@ namespace search::queryeval { */ class ExecuteInfo { public: - ExecuteInfo() : ExecuteInfo(false, 1.0) { } - bool isStrict() const { return _strict; } - double hitRate() const { return _hitRate; } + ExecuteInfo() noexcept : ExecuteInfo(false, 1.0F, nullptr) { } + bool isStrict() const noexcept { return _strict; } + float hitRate() const noexcept { return _hitRate; } + bool soft_doom() const noexcept { return _doom && _doom->soft_doom(); } + const vespalib::Doom * getDoom() const { return _doom; } static const ExecuteInfo TRUE; static const ExecuteInfo FALSE; - static ExecuteInfo create(bool strict); - static ExecuteInfo create(bool strict, double HitRate); + static ExecuteInfo create(bool strict, const ExecuteInfo & org) noexcept { + return {strict, org._hitRate, org.getDoom()}; + } + static ExecuteInfo create(bool strict, const vespalib::Doom * doom) noexcept { + return create(strict, 1.0F, doom); + } + static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom) noexcept { + return {strict, hitRate, doom}; + } + static ExecuteInfo create(bool strict) noexcept { + return create(strict, 1.0F); + } + static ExecuteInfo create(bool strict, float hitRate) noexcept { + return create(strict, hitRate, nullptr); + } private: - ExecuteInfo(bool strict, double hitRate_in) - : _hitRate(hitRate_in), + ExecuteInfo(bool strict, float hitRate_in, const vespalib::Doom * doom) noexcept + : _doom(doom), + _hitRate(hitRate_in), _strict(strict) { } - double _hitRate; - bool _strict; + const vespalib::Doom * _doom; + float _hitRate; + bool _strict; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 9c3910b20f9..16461487525 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -57,7 +57,7 @@ void SameElementBlueprint::fetchPostings(const ExecuteInfo &execInfo) { for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo.hitRate())); + _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 48a09f099a6..eb6241a99d5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -4,7 +4,6 @@ #include "wand_parts.h" #include "parallel_weak_and_search.h" #include -#include #include #include #include @@ -77,10 +76,10 @@ ParallelWeakAndBlueprint::createLeafSearch(const search::fef::TermFieldMatchData const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); // TODO: pass ownership with unique_ptr - terms.push_back(wand::Term(_terms[i]->createSearch(*childrenMatchData, true).release(), - _weights[i], - childState.estimate().estHits, - childState.field(0).resolve(*childrenMatchData))); + terms.emplace_back(_terms[i]->createSearch(*childrenMatchData, true).release(), + _weights[i], + childState.estimate().estHits, + childState.field(0).resolve(*childrenMatchData)); } return SearchIterator::UP (ParallelWeakAndSearch::create(terms, @@ -101,9 +100,9 @@ ParallelWeakAndBlueprint::createFilterSearch(bool strict, FilterConstraint const void ParallelWeakAndBlueprint::fetchPostings(const ExecuteInfo & execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index 4e06f170253..77d9875bf69 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -120,16 +120,16 @@ WeightedSetTermBlueprint::create_matching_elements_search(const MatchingElements if (fields.has_field(_children_field.getName())) { return std::make_unique(*this, _children_field.getName(), _terms); } else { - return std::unique_ptr(); + return {}; } } void WeightedSetTermBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } -- cgit v1.2.3 From 1f744326f621aeb79b44a21deee0d31f9cd0ac79 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 23 Sep 2023 11:28:40 +0000 Subject: - Single filter terms can be lifted out from weighted sets. --- .../attribute_weighted_set_blueprint_test.cpp | 99 ++++++++++++++++------ .../attribute/attribute_weighted_set_blueprint.cpp | 19 +++-- 2 files changed, 81 insertions(+), 37 deletions(-) diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index d7a854e0afc..6c6f05fd5e2 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -29,14 +29,14 @@ using namespace search::attribute::test; namespace { void -setupAttributeManager(MockAttributeManager &manager) +setupAttributeManager(MockAttributeManager &manager, bool isFilter) { AttributeVector::DocId docId; { - AttributeVector::SP attr_sp = AttributeFactory::createAttribute("integer", Config(BasicType("int64"))); + AttributeVector::SP attr_sp = AttributeFactory::createAttribute("integer", Config(BasicType("int64")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - IntegerAttribute *attr = (IntegerAttribute*)(attr_sp.get()); + auto *attr = (IntegerAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -45,10 +45,10 @@ setupAttributeManager(MockAttributeManager &manager) } } { - AttributeVector::SP attr_sp = AttributeFactory::createAttribute("string", Config(BasicType("string"))); + AttributeVector::SP attr_sp = AttributeFactory::createAttribute("string", Config(BasicType("string")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - StringAttribute *attr = (StringAttribute*)(attr_sp.get()); + auto *attr = (StringAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -58,9 +58,9 @@ setupAttributeManager(MockAttributeManager &manager) } { AttributeVector::SP attr_sp = AttributeFactory::createAttribute( - "multi", Config(BasicType("int64"), search::attribute::CollectionType("array"))); + "multi", Config(BasicType("int64"), search::attribute::CollectionType("array")).setIsFilter(isFilter)); manager.addAttribute(attr_sp); - IntegerAttribute *attr = (IntegerAttribute*)(attr_sp.get()); + auto *attr = (IntegerAttribute*)(attr_sp.get()); for (size_t i = 1; i < 10; ++i) { attr->addDoc(docId); assert(i == docId); @@ -78,35 +78,43 @@ struct WS { TermFieldHandle handle; std::vector > tokens; - WS(IAttributeManager & manager) : attribute_manager(manager), layout(), handle(layout.allocTermField(fieldId)), tokens() { + explicit WS(IAttributeManager & manager) + : attribute_manager(manager), + layout(), handle(layout.allocTermField(fieldId)), + tokens() + { MatchData::UP tmp = layout.createMatchData(); ASSERT_TRUE(tmp->resolveTermField(handle)->getFieldId() == fieldId); } WS &add(const std::string &token, uint32_t weight) { - tokens.push_back(std::make_pair(token, weight)); + tokens.emplace_back(token, weight); return *this; } Node::UP createNode() const { - SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); - for (size_t i = 0; i < tokens.size(); ++i) { - node->addTerm(tokens[i].first, Weight(tokens[i].second)); + auto *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); + for (const auto & token : tokens) { + node->addTerm(token.first, Weight(token.second)); } return Node::UP(node); } - bool isGenericSearch(Searchable &searchable, const std::string &field, bool strict) const { + SearchIterator::UP + createSearch(Searchable &searchable, const std::string &field, bool strict) const { AttributeContext ac(attribute_manager); FakeRequestContext requestContext(&ac); MatchData::UP md = layout.createMatchData(); Node::UP node = createNode(); FieldSpecList fields; - fields.add(FieldSpec(field, fieldId, handle)); + fields.add(FieldSpec(field, fieldId, handle, ac.getAttribute(field)->getIsFilter())); queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); bp->fetchPostings(queryeval::ExecuteInfo::create(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); - return (dynamic_cast(sb.get()) != 0); + return sb; + } + bool isWeightedSetTermSearch(Searchable &searchable, const std::string &field, bool strict) const { + return dynamic_cast(createSearch(searchable, field, strict).get()) != nullptr; } FakeResult search(Searchable &searchable, const std::string &field, bool strict) const { @@ -140,23 +148,58 @@ struct WS { } // namespace +void test_tokens(bool isFilter, const std::vector & docs) { + MockAttributeManager manager; + setupAttributeManager(manager, isFilter); + AttributeBlueprintFactory adapter; + + FakeResult expect = FakeResult(); + WS ws = WS(manager); + for (uint32_t doc : docs) { + auto docS = vespalib::stringify(doc); + int32_t weight = doc * 10; + expect.doc(doc).weight(weight).pos(0); + ws.add(docS, weight); + } + + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "integer", true)); + EXPECT_TRUE(!ws.isWeightedSetTermSearch(adapter, "integer", false)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "string", true)); + EXPECT_TRUE(!ws.isWeightedSetTermSearch(adapter, "string", false)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", false)); + + EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", false)); +} TEST("attribute_weighted_set_test") { + test_tokens(false, {3, 5, 7}); + test_tokens(true, {3, 5, 7}); + test_tokens(false, {3}); +} + +TEST("attribute_weighted_set_single_token_filter_lifted_out") { MockAttributeManager manager; - setupAttributeManager(manager); + setupAttributeManager(manager, true); AttributeBlueprintFactory adapter; - FakeResult expect = FakeResult() - .doc(3).elem(0).weight(30).pos(0) - .doc(5).elem(0).weight(50).pos(0) - .doc(7).elem(0).weight(70).pos(0); - WS ws = WS(manager).add("7", 70).add("5", 50).add("3", 30); - - EXPECT_TRUE(ws.isGenericSearch(adapter, "integer", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "integer", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "string", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "string", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", true)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", false)); + FakeResult expect = FakeResult().doc(3).elem(0).weight(30).pos(0); + WS ws = WS(manager).add("3", 30); + + EXPECT_EQUAL("search::FilterAttributeIteratorStrict > >", + ws.createSearch(adapter, "integer", true)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorT > >", + ws.createSearch(adapter, "integer", false)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorStrict >", + ws.createSearch(adapter, "string", true)->getClassName()); + EXPECT_EQUAL("search::FilterAttributeIteratorT >", + ws.createSearch(adapter, "string", false)->getClassName()); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isWeightedSetTermSearch(adapter, "multi", false)); EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index 108128eeb39..94c560a0dae 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -30,7 +30,7 @@ protected: const attribute::IAttributeVector &attribute() const { return _attr; } public: - UseAttr(const attribute::IAttributeVector & attr) + explicit UseAttr(const attribute::IAttributeVector & attr) : _attr(attr) {} }; @@ -40,7 +40,7 @@ class UseStringEnum : public UseAttr { public: using TokenT = uint32_t; - UseStringEnum(const IAttributeVector & attr) + explicit UseStringEnum(const IAttributeVector & attr) : UseAttr(attr) {} auto mapToken(const ISearchContext &context) const { return attribute().findFoldedEnums(context.queryTerm()->getTerm()); @@ -56,7 +56,7 @@ class UseInteger : public UseAttr { public: using TokenT = uint64_t; - UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} + explicit UseInteger(const IAttributeVector & attr) : UseAttr(attr) {} std::vector mapToken(const ISearchContext &context) const { std::vector result; Int64Range range(context.getAsIntegerTerm()); @@ -157,6 +157,10 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr assert(tfmda.size() == 1); assert(getState().numFields() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; + bool field_is_filter = getState().fields()[0].isFilter(); + if (field_is_filter && (_contexts.size() == 1)) { + return _contexts[0]->createIterator(&tfmd, strict); + } if (strict) { // use generic weighted set search fef::MatchDataLayout layout; auto handle = layout.allocTermField(tfmd.getFieldId()); @@ -167,7 +171,6 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr // TODO: pass ownership with unique_ptr children[i] = _contexts[i]->createIterator(child_tfmd, true).release(); } - bool field_is_filter = getState().fields()[0].isFilter(); return queryeval::WeightedSetTermSearch::create(children, tfmd, field_is_filter, _weights, std::move(match_data)); } else { // use attribute filter optimization bool isString = (_attr.isStringType() && _attr.hasEnum()); @@ -182,18 +185,16 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr } queryeval::SearchIterator::UP -AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const +AttributeWeightedSetBlueprint::createFilterSearch(bool strict, FilterConstraint) const { - (void) constraint; std::vector> children; children.reserve(_contexts.size()); for (auto& context : _contexts) { - auto wrapper = std::make_unique(1); + auto wrapper = std::make_unique(1); wrapper->wrap(context->createIterator(wrapper->tfmda()[0], strict)); children.emplace_back(std::move(wrapper)); } - search::queryeval::UnpackInfo unpack_info; - return search::queryeval::OrSearch::create(std::move(children), strict, unpack_info); + return queryeval::OrSearch::create(std::move(children), strict, queryeval::UnpackInfo()); } void -- cgit v1.2.3 From 2f00e6e12c4f09f2b3f4844dcf6f3a3df830cb0a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:10:42 +0000 Subject: Update dependency com.google.errorprone:error_prone_annotations to v2.22.0 --- dependency-versions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 962d666bf6b..8367b9a1161 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -34,7 +34,7 @@ 1.0 1.2 - 2.21.1 + 2.22.0 32.1.2-jre 6.0.0 2.15.2 -- cgit v1.2.3 From 2756c4db7e606a32d42fb5ace08e2d6ffd28c771 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sat, 23 Sep 2023 15:25:55 +0200 Subject: Expect fresh com.google.errorprone:error_prone_annotations:2.22.0 --- maven-plugins/allowed-maven-dependencies.txt | 2 +- vespa-dependencies-enforcer/allowed-maven-dependencies.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 970bb6732a1..e1a1adf3b4d 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -7,7 +7,7 @@ com.fasterxml.jackson.core:jackson-annotations:2.15.2 com.fasterxml.jackson.core:jackson-core:2.15.2 com.fasterxml.jackson.core:jackson-databind:2.15.2 com.github.luben:zstd-jni:1.5.5-5 -com.google.errorprone:error_prone_annotations:2.21.1 +com.google.errorprone:error_prone_annotations:2.22.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.2-jre com.google.inject:guice:6.0.0 diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 2972ea7745e..c97511a02e3 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -25,7 +25,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2 com.github.luben:zstd-jni:1.5.5-5 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.21.1 +com.google.errorprone:error_prone_annotations:2.22.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.2-jre com.google.inject:guice:6.0.0 -- cgit v1.2.3 From e7463c4e6e927989289999f3b436c738a3c8f8e1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 24 Sep 2023 10:32:42 +0000 Subject: Update dependency org.xerial.snappy:snappy-java to v1.1.10.4 --- dependency-versions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 8367b9a1161..90fe48ab0a5 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -123,7 +123,7 @@ 3.24.3 7.3.2 1.3.6 - 1.1.10.3 + 1.1.10.4 3.1.2 3.1.0 2.12.2 -- cgit v1.2.3 From 9b69e8bfd4a02a7b26dfca33f7debfbfc0d05ba9 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 24 Sep 2023 16:22:27 +0200 Subject: Expect org.xerial.snappy:snappy-java:1.1.10.4 --- vespa-dependencies-enforcer/allowed-maven-dependencies.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index c97511a02e3..1f6a15c7c8f 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -187,7 +187,7 @@ org.slf4j:slf4j-api:1.7.36 org.slf4j:slf4j-jdk14:1.7.36 org.slf4j:slf4j-simple:1.7.36 org.tukaani:xz:1.9 -org.xerial.snappy:snappy-java:1.1.10.3 +org.xerial.snappy:snappy-java:1.1.10.4 software.amazon.ion:ion-java:1.0.2 xerces:xercesImpl:2.12.2 -- cgit v1.2.3 From cc7800772354a55059a6aaaf07d3278b91e558cd Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Sun, 24 Sep 2023 18:40:18 +0000 Subject: Simple code cleanup --- .../searchlib/attribute/attribute_blueprint_factory.cpp | 5 +---- .../searchlib/attribute/document_weight_or_filter_search.cpp | 6 +++--- .../searchlib/attribute/document_weight_or_filter_search.h | 6 +++--- .../searchlib/queryeval/weighted_set_term_blueprint.cpp | 12 +++++------- .../vespa/searchlib/queryeval/weighted_set_term_blueprint.h | 2 +- .../vespa/searchlib/queryeval/weighted_set_term_search.cpp | 8 ++++---- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 1519bb14554..b4cdd621b71 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -337,10 +337,7 @@ public: if (tfmda.size() == 1) { // search in exactly one field fef::TermFieldMatchData &tfmd = *tfmda[0]; - return search::common::create_location_iterator(tfmd, - _attribute.getNumDocs(), - strict, - _location); + return common::create_location_iterator(tfmd, _attribute.getNumDocs(), strict, _location); } else { LOG(debug, "wrong size tfmda: %zu (fallback to old location iterator)\n", tfmda.size()); } diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp index 3c0bae00047..e2566c94f1c 100644 --- a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.cpp @@ -11,8 +11,8 @@ class DocumentWeightOrFilterSearchImpl : public DocumentWeightOrFilterSearch { AttributeIteratorPack _children; public: - DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children); - ~DocumentWeightOrFilterSearchImpl(); + explicit DocumentWeightOrFilterSearchImpl(AttributeIteratorPack&& children); + ~DocumentWeightOrFilterSearchImpl() override; void doSeek(uint32_t docId) override; @@ -67,7 +67,7 @@ DocumentWeightOrFilterSearchImpl::doUnpack(uint32_t) { } -std::unique_ptr +std::unique_ptr DocumentWeightOrFilterSearch::create(std::vector&& children) { if (children.empty()) { diff --git a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h index 62be883ab52..c601856573f 100644 --- a/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h +++ b/searchlib/src/vespa/searchlib/attribute/document_weight_or_filter_search.h @@ -9,15 +9,15 @@ namespace search::attribute { * Filter iterator on top of document weight iterators with OR semantics used during * calculation of global filter for weighted set terms, wand terms and dot product terms. */ -class DocumentWeightOrFilterSearch : public search::queryeval::SearchIterator +class DocumentWeightOrFilterSearch : public queryeval::SearchIterator { protected: DocumentWeightOrFilterSearch() - : search::queryeval::SearchIterator() + : queryeval::SearchIterator() { } public: - static std::unique_ptr create(std::vector&& children); + static std::unique_ptr create(std::vector&& children); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index 4e06f170253..72ce6aed36c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -33,10 +33,8 @@ WeightedSetTermMatchingElementsSearch::WeightedSetTermMatchingElementsSearch(con _search() { _tfmda.add(&_tfmd); - auto generic_search = bp.createLeafSearch(_tfmda, false); - auto weighted_set_term_search = dynamic_cast(generic_search.get()); - generic_search.release(); - _search.reset(weighted_set_term_search); + _search.reset(static_cast(bp.createLeafSearch(_tfmda, false).release())); + } WeightedSetTermMatchingElementsSearch::~WeightedSetTermMatchingElementsSearch() = default; @@ -120,7 +118,7 @@ WeightedSetTermBlueprint::create_matching_elements_search(const MatchingElements if (fields.has_field(_children_field.getName())) { return std::make_unique(*this, _children_field.getName(), _terms); } else { - return std::unique_ptr(); + return {}; } } @@ -128,8 +126,8 @@ void WeightedSetTermBlueprint::fetchPostings(const ExecuteInfo &execInfo) { ExecuteInfo childInfo = ExecuteInfo::create(true, execInfo.hitRate()); - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(childInfo); + for (const auto & _term : _terms) { + _term->fetchPostings(childInfo); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h index 0e3c82444d7..9c8d6d88329 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h @@ -18,7 +18,7 @@ class WeightedSetTermBlueprint : public ComplexLeafBlueprint std::vector _terms; public: - WeightedSetTermBlueprint(const FieldSpec &field); + explicit WeightedSetTermBlueprint(const FieldSpec &field); WeightedSetTermBlueprint(const WeightedSetTermBlueprint &) = delete; WeightedSetTermBlueprint &operator=(const WeightedSetTermBlueprint &) = delete; ~WeightedSetTermBlueprint() override; diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp index ee3978705cf..8478a0d3c35 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp @@ -21,7 +21,7 @@ private: struct CmpDocId { const uint32_t *termPos; - CmpDocId(const uint32_t *tp) : termPos(tp) {} + explicit CmpDocId(const uint32_t *tp) : termPos(tp) {} bool operator()(const ref_t &a, const ref_t &b) const { return (termPos[a] < termPos[b]); } @@ -29,7 +29,7 @@ private: struct CmpWeight { const int32_t *weight; - CmpWeight(const int32_t *w) : weight(w) {} + explicit CmpWeight(const int32_t *w) : weight(w) {} bool operator()(const ref_t &a, const ref_t &b) const { return (weight[a] > weight[b]); } @@ -61,7 +61,7 @@ private: } public: - WeightedSetTermSearchImpl(search::fef::TermFieldMatchData &tmd, + WeightedSetTermSearchImpl(fef::TermFieldMatchData &tmd, bool field_is_filter, const std::vector &weights, IteratorPack &&iteratorPack) @@ -180,7 +180,7 @@ WeightedSetTermSearch::create(const std::vector &children, //----------------------------------------------------------------------------- SearchIterator::UP -WeightedSetTermSearch::create(search::fef::TermFieldMatchData &tmd, +WeightedSetTermSearch::create(fef::TermFieldMatchData &tmd, bool field_is_filter, const std::vector &weights, std::vector &&iterators) -- cgit v1.2.3 From 10ab7c2afa03b13d395baf6016fd564cee4d95a2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 06:26:01 +0000 Subject: Update dependency @vitejs/plugin-react to v4.1.0 --- client/js/app/yarn.lock | 177 +++++++++++++++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 61 deletions(-) diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 1c8f628fe18..572c3095570 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -24,9 +24,9 @@ chalk "^2.4.2" "@babel/compat-data@^7.22.9": - version "7.22.9" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.9.tgz#71cdb00a1ce3a329ce4cbec3a44f9fef35669730" - integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.20.tgz#8df6e96661209623f1975d66c35ffca66f3306d0" + integrity sha512-BQYjKbpXjoXwFW5jGqiizJQQT/aC7pFm9Ok1OWssonuguICi264lbgMzRp2ZMmRSlfkX6DsWDDcsrctK8Rwfiw== "@babel/core@^7.1.0", "@babel/core@^7.12.17": version "7.22.9" @@ -70,21 +70,21 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/core@^7.22.9": - version "7.22.11" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.11.tgz#8033acaa2aa24c3f814edaaa057f3ce0ba559c24" - integrity sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ== +"@babel/core@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.22.20.tgz#e3d0eed84c049e2a2ae0a64d27b6a37edec385b7" + integrity sha512-Y6jd1ahLubuYweD/zJH+vvOY141v4f9igNQAQ+MBgq9JlHS2iTsZKn1aMsb3vGccZsXI16VzTBw52Xx0DWmtnA== dependencies: "@ampproject/remapping" "^2.2.0" - "@babel/code-frame" "^7.22.10" - "@babel/generator" "^7.22.10" - "@babel/helper-compilation-targets" "^7.22.10" - "@babel/helper-module-transforms" "^7.22.9" - "@babel/helpers" "^7.22.11" - "@babel/parser" "^7.22.11" - "@babel/template" "^7.22.5" - "@babel/traverse" "^7.22.11" - "@babel/types" "^7.22.11" + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.22.15" + "@babel/helper-compilation-targets" "^7.22.15" + "@babel/helper-module-transforms" "^7.22.20" + "@babel/helpers" "^7.22.15" + "@babel/parser" "^7.22.16" + "@babel/template" "^7.22.15" + "@babel/traverse" "^7.22.20" + "@babel/types" "^7.22.19" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -111,7 +111,7 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/helper-compilation-targets@^7.22.10", "@babel/helper-compilation-targets@^7.22.15": +"@babel/helper-compilation-targets@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" integrity sha512-y6EEzULok0Qvz8yyLkCvVX+02ic+By2UdOhylwUOvOn9dvYc9mKICJuuU1n1XBI02YWsNsnrY1kc6DVbjcXbtw== @@ -133,10 +133,10 @@ lru-cache "^5.1.1" semver "^6.3.1" -"@babel/helper-environment-visitor@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98" - integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q== +"@babel/helper-environment-visitor@^7.22.20", "@babel/helper-environment-visitor@^7.22.5": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== "@babel/helper-function-name@^7.22.5": version "7.22.5" @@ -178,6 +178,17 @@ "@babel/helper-split-export-declaration" "^7.22.6" "@babel/helper-validator-identifier" "^7.22.15" +"@babel/helper-module-transforms@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.20.tgz#da9edc14794babbe7386df438f3768067132f59e" + integrity sha512-dLT7JVWIUUxKOs1UnJUBR3S70YK+pKX6AbJgB2vMIvEkZkrfJDbYDJesnPshtKV4LhDOR3Oc5YULeDizRek+5A== + dependencies: + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-module-imports" "^7.22.15" + "@babel/helper-simple-access" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-module-transforms@^7.22.5": version "7.22.9" resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.22.9.tgz#92dfcb1fbbb2bc62529024f72d942a8c97142129" @@ -213,17 +224,17 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== -"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.5": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.15.tgz#601fa28e4cc06786c18912dca138cec73b882044" - integrity sha512-4E/F9IIEi8WR94324mbDUMo074YTheJmd7eZF5vITTeYchqAi6sYXRLHUVsmkdmY4QjfKTcB2jB7dVP3NaBElQ== +"@babel/helper-validator-identifier@^7.22.15", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.22.5": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== "@babel/helper-validator-option@^7.22.15", "@babel/helper-validator-option@^7.22.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.22.15.tgz#694c30dfa1d09a6534cdfcafbe56789d36aba040" integrity sha512-bMn7RmyFjY/mdECUbgn9eoSY4vqvacUnS9i9vGAGttgFWesO6B4CYWA7XlpbWgBt71iv/hfbPlynohStqnu5hA== -"@babel/helpers@^7.22.11", "@babel/helpers@^7.22.15": +"@babel/helpers@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.22.15.tgz#f09c3df31e86e3ea0b7ff7556d85cdebd47ea6f1" integrity sha512-7pAjK0aSdxOwR+CcYAqgWOGy5dcfvzsTIfFTb2odQqW47MDfv14UaJDY6eng8ylM2EaeKXdxaSWESbkmaQHTmw== @@ -242,11 +253,11 @@ "@babel/types" "^7.22.11" "@babel/highlight@^7.22.13": - version "7.22.13" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.13.tgz#9cda839e5d3be9ca9e8c26b6dd69e7548f0cbf16" - integrity sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ== + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== dependencies: - "@babel/helper-validator-identifier" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" chalk "^2.4.2" js-tokens "^4.0.0" @@ -404,7 +415,7 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17": +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.17": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.17.tgz#b23c203ab3707e3be816043081b4a994fcacec44" integrity sha512-xK4Uwm0JnAMvxYZxOVecss85WxTEIbTa7bnGyf/+EgCL5Zt3U7htUpEOWv9detPlamGKuRzCqw74xVglDWpPdg== @@ -420,6 +431,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.22.15", "@babel/traverse@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.20.tgz#db572d9cb5c79e02d83e5618b82f6991c07584c9" + integrity sha512-eU260mPZbU7mZ0N+X10pxXhQFMGTeLb9eFS0mxehS8HZp9o1uSnFeWQuG1UPrlxgA7QoUzFhOnilHDp0AXCyHw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.22.15" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.22.5" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.22.16" + "@babel/types" "^7.22.19" + debug "^4.1.0" + globals "^11.1.0" + "@babel/traverse@^7.22.8": version "7.22.11" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c" @@ -436,7 +463,16 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.22.15", "@babel/types@^7.22.17", "@babel/types@^7.22.5", "@babel/types@^7.3.3": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.17", "@babel/types@^7.22.19", "@babel/types@^7.22.5": + version "7.22.19" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.19.tgz#7425343253556916e440e662bb221a93ddb75684" + integrity sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.19" + to-fast-properties "^2.0.0" + +"@babel/types@^7.22.10", "@babel/types@^7.22.11", "@babel/types@^7.3.3": version "7.22.17" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.17.tgz#f753352c4610ffddf9c8bc6823f9ff03e2303eee" integrity sha512-YSQPHLFtQNE5xN9tHuZnzu8vPr61wVTBZdfv1meex1NBosa4iT05k/Jw06ddJugi4bk7The/oSwQGFcksmEJQg== @@ -1244,22 +1280,40 @@ "@types/babel__template" "*" "@types/babel__traverse" "*" +"@types/babel__core@^7.20.2": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.2.tgz#215db4f4a35d710256579784a548907237728756" + integrity sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA== + dependencies: + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + "@types/babel__generator" "*" + "@types/babel__template" "*" + "@types/babel__traverse" "*" + "@types/babel__generator@*": - version "7.6.4" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.4.tgz#1f20ce4c5b1990b37900b63f050182d28c2439b7" - integrity sha512-tFkciB9j2K755yrTALxD44McOrk+gfpIpvC3sxHjRawj6PfnQxrse4Clq5y/Rq+G3mrBurMax/lG8Qn2t9mSsg== + version "7.6.5" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.5.tgz#281f4764bcbbbc51fdded0f25aa587b4ce14da95" + integrity sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.4.1" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.1.tgz#3d1a48fd9d6c0edfd56f2ff578daed48f36c8969" - integrity sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g== + version "7.4.2" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.2.tgz#843e9f1f47c957553b0c374481dc4772921d6a6b" + integrity sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" -"@types/babel__traverse@*", "@types/babel__traverse@^7.0.6": +"@types/babel__traverse@*": + version "7.20.2" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.2.tgz#4ddf99d95cfdd946ff35d2b65c978d9c9bf2645d" + integrity sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw== + dependencies: + "@babel/types" "^7.20.7" + +"@types/babel__traverse@^7.0.6": version "7.20.1" resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.1.tgz#dd6f1d2411ae677dcb2db008c962598be31d6acf" integrity sha512-MitHFXnhtgwsGZWtT68URpOvLN4EREih1u3QtQiN4VdAxWKRVvGCSvw/Qth0M0Qq3pJpnGOu5JaM/ydK7OGbqg== @@ -1337,13 +1391,14 @@ "@types/yargs-parser" "*" "@vitejs/plugin-react@^4": - version "4.0.4" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.0.4.tgz#31c3f779dc534e045c4b134e7cf7b150af0a7646" - integrity sha512-7wU921ABnNYkETiMaZy7XqpueMnpu5VxvVps13MjmCo+utBdD79sZzrApHawHtVX66cCJQQTXFcjH0y9dSUK8g== + version "4.1.0" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.1.0.tgz#e4f56f46fd737c5d386bb1f1ade86ba275fe09bd" + integrity sha512-rM0SqazU9iqPUraQ2JlIvReeaxOoRj6n+PzB1C0cBzIbd8qP336nC39/R9yPi3wVcah7E7j/kdU1uCUqMEU4OQ== dependencies: - "@babel/core" "^7.22.9" + "@babel/core" "^7.22.20" "@babel/plugin-transform-react-jsx-self" "^7.22.5" "@babel/plugin-transform-react-jsx-source" "^7.22.5" + "@types/babel__core" "^7.20.2" react-refresh "^0.14.0" acorn-jsx@^5.3.2: @@ -1725,14 +1780,14 @@ braces@^3.0.2: fill-range "^7.0.1" browserslist@^4.21.9: - version "4.21.10" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.10.tgz#dbbac576628c13d3b2231332cb2ec5a46e015bb0" - integrity sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ== + version "4.21.11" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.11.tgz#35f74a3e51adc4d193dcd76ea13858de7b8fecb8" + integrity sha512-xn1UXOKUz7DjdGlg9RrUr0GGiWzI97UQJnugHtH0OLDfJB7jMgoIkYvRIEO1l9EeEERVqeqLYOcFBW9ldjypbQ== dependencies: - caniuse-lite "^1.0.30001517" - electron-to-chromium "^1.4.477" + caniuse-lite "^1.0.30001538" + electron-to-chromium "^1.4.526" node-releases "^2.0.13" - update-browserslist-db "^1.0.11" + update-browserslist-db "^1.0.13" bser@2.1.1: version "2.1.1" @@ -1791,10 +1846,10 @@ camelcase@^6.2.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001517: - version "1.0.30001533" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001533.tgz#1180daeb2518b93c82f19b904d1fefcf82197707" - integrity sha512-9aY/b05NKU4Yl2sbcJhn4A7MsGwR1EPfW/nrqsnqVA0Oq50wpmPaGI+R1Z0UKlUl96oxUkGEOILWtOHck0eCWw== +caniuse-lite@^1.0.30001538: + version "1.0.30001539" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001539.tgz#325a387ab1ed236df2c12dc6cd43a4fff9903a44" + integrity sha512-hfS5tE8bnNiNvEOEkm8HElUHroYwlqMMENEzELymy77+tJ6m+gA2krtHl5hxJaj71OlpC2cHZbdSMX1/YEqEkA== capture-exit@^2.0.0: version "2.0.0" @@ -2124,10 +2179,10 @@ dom-helpers@^5.0.1: "@babel/runtime" "^7.8.7" csstype "^3.0.2" -electron-to-chromium@^1.4.477: - version "1.4.515" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.515.tgz#f5fec9662106ac5752894af221606cf4db443e70" - integrity sha512-VTq6vjk3kCfG2qdzQRd/i9dIyVVm0dbtZIgFzrLgfB73mXDQT2HPKVRc1EoZcAVUv9XhXAu08DWqJuababdGGg== +electron-to-chromium@^1.4.526: + version "1.4.528" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.528.tgz#7c900fd73d9d2e8bb0dab0e301f25f0f4776ef2c" + integrity sha512-UdREXMXzLkREF4jA8t89FQjA8WHI6ssP38PMY4/4KhXFQbtImnghh4GkCgrtiZwLKUKVD2iTVXvDVQjfomEQuA== emittery@^0.13.1: version "0.13.1" @@ -5366,10 +5421,10 @@ untildify@^4.0.0: resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== -update-browserslist-db@^1.0.11: - version "1.0.11" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz#9a2a641ad2907ae7b3616506f4b977851db5b940" - integrity sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA== +update-browserslist-db@^1.0.13: + version "1.0.13" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" + integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg== dependencies: escalade "^3.1.1" picocolors "^1.0.0" -- cgit v1.2.3 From d8b2f3d0162cb13d8bbbb20c62ad51e337f0ce6f Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 21 Sep 2023 15:39:47 +0200 Subject: Remove unused endpoint name building from config-model --- .../model/api/ApplicationClusterEndpoint.java | 60 +--------------------- .../container/ApplicationContainerCluster.java | 60 ++++++---------------- .../model/container/ContainerClusterTest.java | 59 ++++----------------- 3 files changed, 28 insertions(+), 151 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 69749ee6f96..42bebf83d55 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -2,15 +2,8 @@ package com.yahoo.config.model.api; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.SystemName; - import java.util.List; import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Represents one endpoint for an application cluster @@ -156,8 +149,6 @@ public class ApplicationClusterEndpoint { public static class DnsName implements Comparable { - private static final int MAX_LABEL_LENGTH = 63; - private final String name; private DnsName(String name) { @@ -168,59 +159,10 @@ public class ApplicationClusterEndpoint { return name; } - public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(systemName, cluster, applicationId) - .filter(Objects::nonNull) // remove null values that were "default" - .collect(Collectors.joining("--")); - return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label - } - - public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(systemName, cluster, applicationId) - .filter(Objects::nonNull) // remove null values that were "default" - .map(DnsName::sanitize) - .collect(Collectors.joining(".")); - return new DnsName(name + suffix); - } - public static DnsName from(String name) { return new DnsName(name); } - private static Stream dnsParts(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId) { - return Stream.of( - nullIfDefault(cluster.value()), - systemPart(systemName), - nullIfDefault(applicationId.instance().value()), - applicationId.application().value(), - applicationId.tenant().value() - ); - } - - /** - * Remove any invalid characters from the hostnames - */ - private static String sanitize(String id) { - return shortenIfNeeded(id.toLowerCase() - .replace('_', '-') - .replaceAll("[^a-z0-9-]*", "")); - } - - /** - * Truncate the given string at the front so its length does not exceed 63 characters. - */ - private static String shortenIfNeeded(String id) { - return id.substring(Math.max(0, id.length() - MAX_LABEL_LENGTH)); - } - - private static String nullIfDefault(String string) { - return Optional.of(string).filter(s -> !s.equals("default")).orElse(null); - } - - private static String systemPart(SystemName systemName) { - return "cd".equals(systemName.value()) ? systemName.value() : null; - } - @Override public String toString() { return "DnsName{" + @@ -232,5 +174,7 @@ public class ApplicationClusterEndpoint { public int compareTo(DnsName o) { return name.compareTo(o.name); } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index b9021912244..1f7351d916c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -10,12 +10,10 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; -import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -203,49 +201,23 @@ public final class ApplicationContainerCluster extends ContainerCluster hosts = getContainers().stream().map(AbstractService::getHostName).sorted().toList(); List endpoints = new ArrayList<>(); - - List hosts = getContainers().stream() - .map(AbstractService::getHostName) - .sorted() - .toList(); - - Set endpointsFromController = deployState.getEndpoints(); - // Add zone-scoped endpoints if not provided by the controller - // TODO(mpolden): Remove this when controller always includes zone-scope endpoints, and config models < 8.230 are gone - if (endpointsFromController.stream().noneMatch(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.zone)) { - for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { - ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( - deployState.zone().system(), - ClusterSpec.Id.from(getName()), - deployState.getProperties().applicationId(), - suffix); - endpoints.add(ApplicationClusterEndpoint.builder() - .zoneScope() - .sharedL4Routing() - .dnsName(l4Name) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ApplicationClusterEndpoint.AuthMethod.mtls) - .build()); - } - } - - // Include all endpoints provided by controller - endpointsFromController.stream() - .filter(ce -> ce.clusterId().equals(getName())) - .forEach(ce -> ce.names().forEach( - name -> endpoints.add(ApplicationClusterEndpoint.builder() - .scope(ce.scope()) - .weight(ce.weight().orElse(1)) // Default to weight=1 if not set - .routingMethod(ce.routingMethod()) - .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ce.authMethod()) - .build()) - )); - this.endpoints = List.copyOf(endpoints); + deployState.getEndpoints().stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .weight(ce.weight().orElse(1)) + .routingMethod(ce.routingMethod()) + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ce.authMethod()) + .build()) + )); + this.endpoints = Collections.unmodifiableList(endpoints); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 894fc55c014..3bdb60a0a8d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -42,14 +42,14 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; -import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Simon Thoresen Hult @@ -365,62 +365,23 @@ public class ContainerClusterTest { @Test void generatesCorrectRoutingInfo() { - // main system: assertNames(main, ApplicationId.from("t1", "a1", "i1"), - Set.of(), + Set.of(new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.i1.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar")), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - - // cd system: - assertNames(cd, - ApplicationId.from("t1", "a1", "i1"), - Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), - new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - } private void assertNames(SystemName systemName, ApplicationId appId, Set globalEndpoints, List expectedSharedL4Names) { -- cgit v1.2.3 From 7ce86b52dcd1deadfca07355b8a47714a471e430 Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Mon, 25 Sep 2023 09:12:10 +0200 Subject: Default to instance when serializing instance_id dimension --- .../configserver/flags/http/FlagsHandlerTest.java | 6 +++--- .../api/systemflags/v1/SystemFlagsDataArchive.java | 18 +++++++++--------- .../api/systemflags/v1/SystemFlagsDataArchiveTest.java | 8 ++++---- .../maintenance/CertificatePoolMaintainerTest.java | 8 -------- .../restapi/user/UserFlagsSerializerTest.java | 6 +++--- .../com/yahoo/vespa/flags/json/DimensionHelper.java | 2 +- 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java index 3b24e1c1b8d..4f8d42e895b 100644 --- a/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java +++ b/configserver-flags/src/test/java/com/yahoo/vespa/configserver/flags/http/FlagsHandlerTest.java @@ -111,7 +111,7 @@ public class FlagsHandlerTest { }, { "type": "blacklist", - "dimension": "application", + "dimension": "instance", "values": [ "app1", "app2" ] } ], @@ -127,7 +127,7 @@ public class FlagsHandlerTest { // GET on id2 should now return what was put verifySuccessfulRequest(Method.GET, "/data/" + FLAG2.id(), "", - "{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); + "{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}"); // The list of flag data should return id1 and id2 verifySuccessfulRequest(Method.GET, "/data", @@ -153,7 +153,7 @@ public class FlagsHandlerTest { // Get all recursivelly displays all flag data verifySuccessfulRequest(Method.GET, "/data?recursive=true", "", - "{\"flags\":[{\"id\":\"id1\",\"rules\":[{\"value\":false}]},{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}]}"); + "{\"flags\":[{\"id\":\"id1\",\"rules\":[{\"value\":false}]},{\"id\":\"id2\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"host1\",\"host2\"]},{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"app1\",\"app2\"]}],\"value\":true}],\"attributes\":{\"zone\":\"zone1\"}}]}"); // Deleting both flags verifySuccessfulRequest(Method.DELETE, "/data/" + FLAG1.id(), "", ""); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index e661c88e117..856af9f4132 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -258,15 +258,15 @@ public class SystemFlagsDataArchive { root = mapper.readTree(fileContent); // TODO (mortent): Remove this after completing migration of APPLICATION_ID dimension // replace "application" with "instance" for all dimension fields -// List dimensionParents = root.findParents("dimension"); -// for (JsonNode parentNode : dimensionParents) { -// JsonNode dimension = parentNode.get("dimension"); -// if (dimension.isTextual() && "application".equals(dimension.textValue())) { -// ObjectNode parent = (ObjectNode) parentNode; -// parent.remove("dimension"); -// parent.put("dimension", "instance"); -// } -// } + List dimensionParents = root.findParents("dimension"); + for (JsonNode parentNode : dimensionParents) { + JsonNode dimension = parentNode.get("dimension"); + if (dimension.isTextual() && "application".equals(dimension.textValue())) { + ObjectNode parent = (ObjectNode) parentNode; + parent.remove("dimension"); + parent.put("dimension", "instance"); + } + } } catch (JsonProcessingException e) { throw new FlagValidationException("Invalid JSON: " + e.getMessage()); } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index aba6cfbfeac..373f8ba9de2 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -245,7 +245,7 @@ public class SystemFlagsDataArchiveTest { "conditions": [ { "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": [ "f:o:o" ] } ], @@ -287,7 +287,7 @@ public class SystemFlagsDataArchiveTest { { "comment": "bar", "type": "whitelist", - "dimension": "application", + "dimension": "instance", "values": [ "f:o:o" ] } ], @@ -308,7 +308,7 @@ public class SystemFlagsDataArchiveTest { @Test void normalize_json_succeed_on_valid_values() { addFile(Condition.Type.WHITELIST, "application", "a:b:c"); -// addFile(Condition.Type.WHITELIST, "instance", "a:b:c"); + addFile(Condition.Type.WHITELIST, "instance", "a:b:c"); addFile(Condition.Type.WHITELIST, "cloud", "yahoo"); addFile(Condition.Type.WHITELIST, "cloud", "aws"); addFile(Condition.Type.WHITELIST, "cloud", "gcp"); @@ -362,7 +362,7 @@ public class SystemFlagsDataArchiveTest { @Test void normalize_json_fail_on_invalid_values() { - failAddFile(Condition.Type.WHITELIST, "application", "a.b.c", "In file flags/temporary/foo/default.json: Invalid application 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); + failAddFile(Condition.Type.WHITELIST, "application", "a.b.c", "In file flags/temporary/foo/default.json: Invalid instance 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); failAddFile(Condition.Type.WHITELIST, "cloud", "foo", "In file flags/temporary/foo/default.json: Unknown cloud: foo"); // cluster-id: any String is valid failAddFile(Condition.Type.WHITELIST, "cluster-type", "foo", "In file flags/temporary/foo/default.json: Invalid cluster-type 'foo' in whitelist condition: Illegal cluster type 'foo'"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java index 88c5ae9ff06..4257261b09b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java @@ -53,12 +53,4 @@ public class CertificatePoolMaintainerTest { assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(n, tester.curator().readUnassignedCertificates().size()); } - - void old_unassigned_certs_are_refreshed() { - tester.flagSource().withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), 1); - assertNumCerts(1); - EndpointCertificateProviderMock endpointCertificateProvider = (EndpointCertificateProviderMock) tester.controller().serviceRegistry().endpointCertificateProvider(); - var request = endpointCertificateProvider.listCertificates().get(0); - - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java index 779aee73dae..eb3f9daef53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserFlagsSerializerTest.java @@ -63,7 +63,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\"}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions are empty since this user is not authorized for any tenants - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\"}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of(), false, email1); @@ -72,7 +72,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"tenant\",\"values\":[\"tenant1\"]}],\"value\":{\"integer\":456,\"string\":\"xyz\"}},{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Resolved for email // Resolved for email, but conditions have filtered out tenant2 - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\"]},{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"value1\"}]}]}", // resolved for email flagData, Set.of("tenant1"), false, email1); @@ -81,7 +81,7 @@ public class UserFlagsSerializerTest { "{\"id\":\"int-id\",\"rules\":[{\"value\":456}]}," + // Default from DB "{\"id\":\"jackson-id\",\"rules\":[{\"value\":{\"integer\":123,\"string\":\"abc\"}}]}," + // Default from code, no DB values match // Includes last value from DB which is not conditioned on email and the default from code - "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"application\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + + "{\"id\":\"list-id\",\"rules\":[{\"conditions\":[{\"type\":\"blacklist\",\"dimension\":\"instance\",\"values\":[\"tenant1:video:default\",\"tenant1:video:default\",\"tenant2:music:default\"]}],\"value\":[\"value1\",\"value3\"]},{\"value\":[\"a\"]}]}," + "{\"id\":\"string-id\",\"rules\":[{\"value\":\"default value\"}]}]}", // Default from code flagData, Set.of(), true, "operator@domain.tld"); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index f867daac245..8fb48c8a82f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -23,7 +23,7 @@ public class DimensionHelper { serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, List.of("console-user-email")); serializedDimensions.put(FetchVector.Dimension.ENVIRONMENT, List.of("environment")); serializedDimensions.put(FetchVector.Dimension.HOSTNAME, List.of("hostname")); - serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, List.of("application", "instance")); + serializedDimensions.put(FetchVector.Dimension.INSTANCE_ID, List.of("instance", "application")); serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, List.of("node-type")); serializedDimensions.put(FetchVector.Dimension.SYSTEM, List.of("system")); serializedDimensions.put(FetchVector.Dimension.TENANT_ID, List.of("tenant")); -- cgit v1.2.3 From 289706aee0228abca28aee0407701bb6e90675b4 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 25 Sep 2023 09:12:37 +0200 Subject: Convert to record --- .../config/model/api/ApplicationClusterEndpoint.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 42bebf83d55..215439aa42a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -147,13 +147,7 @@ public class ApplicationClusterEndpoint { } - public static class DnsName implements Comparable { - - private final String name; - - private DnsName(String name) { - this.name = name; - } + public record DnsName(String name) implements Comparable { public String value() { return name; @@ -163,13 +157,6 @@ public class ApplicationClusterEndpoint { return new DnsName(name); } - @Override - public String toString() { - return "DnsName{" + - "name='" + name + '\'' + - '}'; - } - @Override public int compareTo(DnsName o) { return name.compareTo(o.name); -- cgit v1.2.3 From 8a24f52e3b42c17c6c63ebebde5ad89df48851a1 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 25 Sep 2023 09:15:54 +0200 Subject: Revert "Revert "Validate use of s3 urls in config"" --- .../application/validation/UrlConfigValidator.java | 52 ++++++++++ .../model/application/validation/Validation.java | 1 + .../filedistribution/UserConfiguredFiles.java | 5 +- .../validation/UrlConfigValidatorTest.java | 107 +++++++++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java create mode 100644 config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java new file mode 100644 index 00000000000..e6cd1a9e192 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; + +import java.util.Optional; + +/** + * Validates that config using s3:// urls is used in public system and with nodes that are exclusive. + * + * @author hmusum + */ +public class UrlConfigValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState state) { + if (! state.isHostedTenantApplication(model.getAdmin().getApplicationType())) return; + + model.getContainerClusters().forEach((__, cluster) -> { + var isExclusive = hasExclusiveNodes(model, cluster); + validateS3UlsInConfig(state, cluster, isExclusive); + }); + } + + private static boolean hasExclusiveNodes(VespaModel model, ApplicationContainerCluster cluster) { + return model.hostSystem().getHosts() + .stream() + .flatMap(hostResource -> hostResource.spec().membership().stream()) + .filter(membership -> membership.cluster().id().equals(cluster.id())) + .anyMatch(membership -> membership.cluster().isExclusive()); + } + + private static void validateS3UlsInConfig(DeployState state, ApplicationContainerCluster cluster, boolean isExclusive) { + if (hasUrlInConfig(state)) { + // TODO: Would be even better if we could add which config/field the url is set for in the error message + String message = "Found s3:// urls in config for container cluster " + cluster.getName(); + if ( ! state.zone().system().isPublic()) + throw new IllegalArgumentException(message + ". This is only supported in public systems"); + else if ( ! isExclusive) + throw new IllegalArgumentException(message + ". Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes"); + } + } + + private static boolean hasUrlInConfig(DeployState state) { + return state.getFileRegistry().export().stream() + .anyMatch(fileReference -> fileReference.relativePath.startsWith("s3://")); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 53a553ee624..90616c99979 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -87,6 +87,7 @@ public class Validation { new AccessControlFilterExcludeValidator().validate(model, deployState); new CloudUserFilterValidator().validate(model, deployState); new CloudHttpConnectorValidator().validate(model, deployState); + new UrlConfigValidator().validate(model, deployState); additionalValidators.forEach(v -> v.validate(model, deployState)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 8bed5e64bf5..8d8b4d72f4d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -133,7 +133,10 @@ public class UserConfiguredFiles implements Serializable { Path path; if (isModelType) { var modelReference = ModelReference.valueOf(builder.getValue()); - if (modelReference.path().isEmpty()) return; + if (modelReference.path().isEmpty()) { + modelReference.url().ifPresent(url -> fileRegistry.addUri(url.value())); + return; + } path = Path.fromString(modelReference.path().get().value()); } else { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java new file mode 100644 index 00000000000..cef4d8c27dd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UrlConfigValidatorTest.java @@ -0,0 +1,107 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.embedding.BertBaseEmbedderConfig; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.config.provision.Environment.prod; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class UrlConfigValidatorTest { + + @Test + void failsWhenContainerNodesNotExclusive() throws IOException, SAXException { + runValidatorOnApp(true, SystemName.Public); // Exclusive nodes in public => success + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. This is only supported in public systems", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(true, SystemName.main)) + .getMessage()); + + assertEquals("Found s3:// urls in config for container cluster default. Nodes in the cluster need to be 'exclusive'," + + " see https://cloud.vespa.ai/en/reference/services#nodes", + assertThrows(IllegalArgumentException.class, + () -> runValidatorOnApp(false, SystemName.Public)) + .getMessage()); + } + + private static String containerXml(boolean isExclusive) { + return """ + + + + + + + + + + + + """.formatted(Boolean.toString(isExclusive)); + } + + private static void runValidatorOnApp(boolean isExclusive, SystemName systemName) throws IOException, SAXException { + String container = containerXml(isExclusive); + String servicesXml = """ + + %s + + """.formatted(container); + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + DeployState deployState = createDeployState(app, systemName); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + new UrlConfigValidator().validate(model, deployState); + } + + private static DeployState createDeployState(ApplicationPackage app, SystemName systemName) { + boolean isHosted = true; + var builder = new DeployState.Builder() + .applicationPackage(app) + .zone(new Zone(systemName, prod, RegionName.from("us-east-3"))) + .properties(new TestProperties().setHostedVespa(isHosted)) + .fileRegistry(new MockFileRegistry()); + + Map defs = new HashMap<>(); + defs.put(new ConfigDefinitionKey(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(BertBaseEmbedderConfig.CONFIG_DEF_NAME, BertBaseEmbedderConfig.CONFIG_DEF_SCHEMA)); + builder.configDefinitionRepo(new ConfigDefinitionRepo() { + @Override + public Map getConfigDefinitions() { + return defs; + } + + @Override + public com.yahoo.vespa.config.buildergen.ConfigDefinition get(ConfigDefinitionKey key) { + return defs.get(key); + } + }); + return builder.build(); + } + +} -- cgit v1.2.3 From 9cb5024db5351c01f4ada6d6c6f9a0ab21c975b1 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 25 Sep 2023 09:39:19 +0200 Subject: Update test --- .../com/yahoo/vespa/config/server/model/LbServicesProducerTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index ca2f9da3273..8d1b0fefbaf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -50,10 +50,11 @@ import static org.junit.Assert.assertTrue; public class LbServicesProducerTest { private static final Set endpoints = Set.of( + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint1.suffix")), + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint2.suffix")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); - private static final List zoneDnsSuffixes = List.of(".endpoint1.suffix", ".endpoint2.suffix"); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -228,7 +229,7 @@ public class LbServicesProducerTest { private TestProperties getTestproperties(ApplicationId applicationId) { return new TestProperties() .setHostedVespa(true) - .setZoneDnsSuffixes(zoneDnsSuffixes) .setApplicationId(applicationId); } + } -- cgit v1.2.3 From 06a314c1620dfbbbb9faf79e84d4ed3bd3459f9a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 06:25:36 +0000 Subject: Update dependency org.apache.maven.plugins:maven-shade-plugin to v3.5.1 --- dependency-versions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 90fe48ab0a5..120c6104793 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -153,7 +153,7 @@ ${maven-core.vespa.version} 3.9.0 3.3.1 - 3.5.0 + 3.5.1 3.12.1 3.3.0 1.2.0 -- cgit v1.2.3 From 50c55cd43881e447c7c99705cd04205653323ffe Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 25 Sep 2023 09:40:23 +0200 Subject: Expect maven-shade-plugin:3.5.1 --- maven-plugins/allowed-maven-dependencies.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index e1a1adf3b4d..6853632ea40 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -35,7 +35,7 @@ org.apache.maven:maven-settings-builder:3.9.4 org.apache.maven.enforcer:enforcer-api:3.4.1 org.apache.maven.enforcer:enforcer-rules:3.4.1 org.apache.maven.plugin-tools:maven-plugin-annotations:3.9.0 -org.apache.maven.plugins:maven-shade-plugin:3.5.0 +org.apache.maven.plugins:maven-shade-plugin:3.5.1 org.apache.maven.resolver:maven-resolver-api:1.9.14 org.apache.maven.resolver:maven-resolver-impl:1.9.14 org.apache.maven.resolver:maven-resolver-named-locks:1.9.14 @@ -63,7 +63,7 @@ org.ow2.asm:asm-tree:9.5 org.slf4j:slf4j-api:1.7.36 org.tukaani:xz:1.9 org.twdata.maven:mojo-executor:2.4.0 -org.vafer:jdependency:2.8.0 +org.vafer:jdependency:2.9.0 #[test-only] # Contains dependencies that are used exclusively in 'test' scope -- cgit v1.2.3 From ce541bca4f4021d2b10c17c2a82e9bcd597428a3 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Thu, 21 Sep 2023 18:13:53 +0200 Subject: Encapsulate wiregaurd key + timestamp in new class - Use 'wireguard' object with key and timestamp for Rest api. - Keep zk node format unchanged. --- .../provision/WireguardKeyWithTimestamp.java | 39 ++++++++ .../configserver/noderepository/NodeSpec.java | 40 +++----- .../noderepository/RealNodeRepository.java | 65 +++++++++---- .../bindings/GetWireguardResponse.java | 28 +++--- .../bindings/NodeRepositoryNode.java | 20 +++- .../hosted/node/admin/wireguard/WireguardPeer.java | 6 +- .../noderepository/RealNodeRepositoryTest.java | 16 ++-- .../node/admin/wireguard/WireguardPeerTest.java | 4 +- .../com/yahoo/vespa/hosted/provision/Node.java | 106 +++++++-------------- .../hosted/provision/persistence/CuratorDb.java | 2 +- .../provision/persistence/NodeSerializer.java | 17 +++- .../hosted/provision/restapi/NodePatcher.java | 14 ++- .../hosted/provision/restapi/NodesResponse.java | 15 ++- .../provision/restapi/WireguardResponse.java | 19 ++-- .../provision/testutils/MockNodeRepository.java | 9 +- .../hosted/provision/restapi/responses/cfg1.json | 6 +- .../provision/restapi/responses/docker-node2.json | 4 + .../provision/restapi/responses/node4-wg.json | 4 + .../provision/restapi/responses/wireguard.json | 6 +- 19 files changed, 246 insertions(+), 174 deletions(-) create mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java new file mode 100644 index 00000000000..ecc1cf71113 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKeyWithTimestamp.java @@ -0,0 +1,39 @@ +package com.yahoo.config.provision; + +import com.yahoo.jdisc.Timer; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Random; + +/** + * @author gjoranv + */ +public record WireguardKeyWithTimestamp(WireguardKey key, Instant timestamp) { + + public static final int KEY_ROTATION_BASE = 60; + public static final int KEY_ROTATION_VARIANCE = 10; + public static final int KEY_EXPIRY = KEY_ROTATION_BASE + KEY_ROTATION_VARIANCE + 5; + + public WireguardKeyWithTimestamp { + if (key == null) throw new IllegalArgumentException("Wireguard key cannot be null"); + if (timestamp == null) timestamp = Instant.EPOCH; + } + + public static WireguardKeyWithTimestamp from(String key, long msTimestamp) { + return new WireguardKeyWithTimestamp(WireguardKey.from(key), Instant.ofEpochMilli(msTimestamp)); + } + + public boolean isDueForRotation(Timer timer, ChronoUnit unit, Random random) { + return timer.currentTime().isAfter(keyRotationDueAt(unit, random)); + } + + public boolean hasExpired(Timer timer, ChronoUnit unit) { + return timer.currentTime().isAfter(timestamp.plus(KEY_EXPIRY, unit)); + } + + private Instant keyRotationDueAt(ChronoUnit unit, Random random) { + return timestamp.plus(KEY_ROTATION_BASE + random.nextInt(KEY_ROTATION_VARIANCE), unit); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 0300d7e92ff..d902fb7b3c4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import java.net.URI; @@ -73,9 +74,7 @@ public class NodeSpec { private final List trustStore; - private final Optional wireguardPubkey; - - private final Optional wireguardKeyTimestamp; + private final Optional wireguardKeyWithTimestamp; private final boolean wantToRebuild; @@ -112,8 +111,7 @@ public class NodeSpec { Optional archiveUri, Optional exclusiveTo, List trustStore, - Optional wireguardPubkey, - Optional wireguardKeyTimestamp, + Optional wireguardPubkey, boolean wantToRebuild) { if (state == NodeState.active) { @@ -157,8 +155,7 @@ public class NodeSpec { this.archiveUri = Objects.requireNonNull(archiveUri); this.exclusiveTo = Objects.requireNonNull(exclusiveTo); this.trustStore = Objects.requireNonNull(trustStore); - this.wireguardPubkey = Objects.requireNonNull(wireguardPubkey); - this.wireguardKeyTimestamp = Objects.requireNonNull(wireguardKeyTimestamp); + this.wireguardKeyWithTimestamp = Objects.requireNonNull(wireguardPubkey); this.wantToRebuild = wantToRebuild; } @@ -313,9 +310,7 @@ public class NodeSpec { return trustStore; } - public Optional wireguardPubkey() { return wireguardPubkey; } - - public Optional wireguardKeyTimestamp() { return wireguardKeyTimestamp; } + public Optional wireguardKeyWithTimestamp() { return wireguardKeyWithTimestamp; } public boolean wantToRebuild() { return wantToRebuild; @@ -358,8 +353,7 @@ public class NodeSpec { Objects.equals(archiveUri, that.archiveUri) && Objects.equals(exclusiveTo, that.exclusiveTo) && Objects.equals(trustStore, that.trustStore) && - Objects.equals(wireguardPubkey, that.wireguardPubkey) && - Objects.equals(wireguardKeyTimestamp, that.wireguardKeyTimestamp) && + Objects.equals(wireguardKeyWithTimestamp, that.wireguardKeyWithTimestamp) && Objects.equals(wantToRebuild, that.wantToRebuild); } @@ -398,8 +392,7 @@ public class NodeSpec { archiveUri, exclusiveTo, trustStore, - wireguardPubkey, - wireguardKeyTimestamp, + wireguardKeyWithTimestamp, wantToRebuild); } @@ -438,8 +431,7 @@ public class NodeSpec { + " archiveUri=" + archiveUri + " exclusiveTo=" + exclusiveTo + " trustStore=" + trustStore - + " wireguardPubkey=" + wireguardPubkey - + " wireguardKeyTimestamp=" + wireguardKeyTimestamp + + " wireguardPubkey=" + wireguardKeyWithTimestamp + " wantToRebuild=" + wantToRebuild + " }"; } @@ -477,8 +469,7 @@ public class NodeSpec { private Optional archiveUri = Optional.empty(); private Optional exclusiveTo = Optional.empty(); private List trustStore = List.of(); - private Optional wireguardPubkey = Optional.empty(); - private Optional wireguardKeyTimestamp = Optional.empty(); + private Optional wireguardPubkey = Optional.empty(); private boolean wantToRebuild = false; public Builder() {} @@ -514,8 +505,7 @@ public class NodeSpec { node.archiveUri.ifPresent(this::archiveUri); node.exclusiveTo.ifPresent(this::exclusiveTo); trustStore(node.trustStore); - node.wireguardPubkey.ifPresent(this::wireguardPubkey); - node.wireguardKeyTimestamp.ifPresent(this::wireguardKeyTimestamp); + node.wireguardKeyWithTimestamp.ifPresent(this::wireguardKeyWithTimestamp); wantToRebuild(node.wantToRebuild); } @@ -704,13 +694,13 @@ public class NodeSpec { return this; } - public Builder wireguardPubkey(WireguardKey wireguardPubKey) { - this.wireguardPubkey = Optional.of(wireguardPubKey); + public Builder wireguardPubkey(WireguardKey wireguardPubkey) { + this.wireguardPubkey = Optional.of(new WireguardKeyWithTimestamp(wireguardPubkey, Instant.EPOCH)); return this; } - public Builder wireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - this.wireguardKeyTimestamp = Optional.of(wireguardKeyTimestamp); + public Builder wireguardKeyWithTimestamp(WireguardKeyWithTimestamp wireguardPubKey) { + this.wireguardPubkey = Optional.of(wireguardPubKey); return this; } @@ -846,7 +836,7 @@ public class NodeSpec { wantedFirmwareCheck, currentFirmwareCheck, modelName, resources, realResources, ipAddresses, additionalIpAddresses, reports, events, parentHostname, archiveUri, exclusiveTo, trustStore, - wireguardPubkey, wireguardKeyTimestamp, wantToRebuild); + wireguardPubkey, wantToRebuild); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index a9cc2d698e9..17d3b51398f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; @@ -139,26 +140,28 @@ public class RealNodeRepository implements NodeRepository { return response.nodes.stream() .mapMulti((NodeRepositoryNode node, Consumer consumer) -> { - if (node.wireguardPubkey == null || node.wireguardPubkey.isEmpty()) return; - List ipAddresses = node.ipAddresses.stream() - .map(InetAddresses::forString) - .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) - .map(VersionedIpAddress::from) - .toList(); - if (ipAddresses.isEmpty()) return; + var keyWithTimestamp = createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, + node.wireguardPubkey, + node.wireguardKeyTimestamp); + if (keyWithTimestamp == null) return; - // Unbox to prevent NPE - long keyTimestamp = node.wireguardKeyTimestamp == null ? 0L : node.wireguardKeyTimestamp; + List ipAddresses = getIpAddresses(node); + if (ipAddresses.isEmpty()) return; - consumer.accept(new WireguardPeer(HostName.of(node.hostname), - ipAddresses, - WireguardKey.from(node.wireguardPubkey), - Instant.ofEpochMilli(keyTimestamp))); + consumer.accept(new WireguardPeer(HostName.of(node.hostname), ipAddresses, keyWithTimestamp)); }) .sorted() .toList(); } + private static List getIpAddresses(NodeRepositoryNode node) { + return node.ipAddresses.stream() + .map(InetAddresses::forString) + .filter(address -> !address.isLoopbackAddress() && !address.isLinkLocalAddress() && !address.isSiteLocalAddress()) + .map(VersionedIpAddress::from) + .toList(); + } + @Override public List getConfigserverPeers() { GetWireguardResponse response = configServerApi.get("/nodes/v2/wireguard", GetWireguardResponse.class); @@ -246,8 +249,9 @@ public class RealNodeRepository implements NodeRepository { Optional.ofNullable(node.archiveUri).map(URI::create), Optional.ofNullable(node.exclusiveTo).map(ApplicationId::fromSerializedForm), trustStore, - Optional.ofNullable(node.wireguardPubkey).map(WireguardKey::from), - Optional.ofNullable(node.wireguardKeyTimestamp).map(Instant::ofEpochMilli), + Optional.ofNullable(createWireguardKeyWithTimestamp(node.wireguardKeyWithTimestamp, + node.wireguardPubkey, + node.wireguardKeyTimestamp)), node.wantToRebuild); } @@ -364,20 +368,39 @@ public class RealNodeRepository implements NodeRepository { node.trustStore = nodeAttributes.getTrustStore().stream() .map(item -> new NodeRepositoryNode.TrustStoreItem(item.fingerprint(), item.expiry().toEpochMilli())) .toList(); - node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); + // This is used for patching, and timestamp must only be set on the server side, hence sending EPOCH. + node.wireguardKeyWithTimestamp = nodeAttributes.getWireguardPubkey() + .map(key -> new NodeRepositoryNode.WireguardKeyWithTimestamp(key.value(), 0L)) + .orElse(null); Map reports = nodeAttributes.getReports(); node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports); + // TODO wg: remove when all nodes are using new key+timestamp format + node.wireguardPubkey = nodeAttributes.getWireguardPubkey().map(WireguardKey::value).orElse(null); return node; } private static WireguardPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) { - // Unbox to prevent NPE - long keyTimestamp = configServer.wireguardKeyTimestamp == null ? 0L : configServer.wireguardKeyTimestamp; - return new WireguardPeer(HostName.of(configServer.hostname), configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(), - WireguardKey.from(configServer.wireguardPubkey), - Instant.ofEpochMilli(keyTimestamp)); + createWireguardKeyWithTimestamp(configServer.wireguardKeyWithTimestamp, + configServer.wireguardPubkey, + configServer.wireguardKeyTimestamp)); + } + + private static WireguardKeyWithTimestamp createWireguardKeyWithTimestamp(NodeRepositoryNode.WireguardKeyWithTimestamp wirguardJson, + String oldKeyJson, Long oldTimestampJson) { + if (wirguardJson != null && wirguardJson.key != null && ! wirguardJson.key.isEmpty()) { + return new WireguardKeyWithTimestamp(WireguardKey.from(wirguardJson.key), + Instant.ofEpochMilli(wirguardJson.timestamp)); + // TODO wg: remove when all nodes are using new key+timestamp format + } else if (oldKeyJson != null) { + var timestamp = oldTimestampJson != null ? oldTimestampJson : 0L; + return new WireguardKeyWithTimestamp(WireguardKey.from(oldKeyJson), + Instant.ofEpochMilli(timestamp)); + // TODO END + } else return null; + } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java index dcbf4cc163f..47903795ef7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java @@ -27,27 +27,23 @@ public class GetWireguardResponse { public static class Configserver { @JsonProperty("hostname") - public final String hostname; + public String hostname; @JsonProperty("ipAddresses") - public final List ipAddresses; + public List ipAddresses; + + @JsonProperty("wireguard") + public NodeRepositoryNode.WireguardKeyWithTimestamp wireguardKeyWithTimestamp; - @JsonProperty("wireguardPubkey") - public final String wireguardPubkey; + // TODO wg: remove when all nodes use new key+timestamp format + @JsonProperty("wireguardPubkey") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public String wireguardPubkey; @JsonProperty("wireguardKeyTimestamp") - public final Long wireguardKeyTimestamp; - - @JsonCreator - public Configserver(@JsonProperty("hostname") String hostname, - @JsonProperty("ipAddresses") List ipAddresses, - @JsonProperty("wireguardPubkey") String wireguardPubkey, - @JsonProperty("wireguardKeyTimestamp") Long wireguardKeyTimestamp) { - this.hostname = hostname; - this.ipAddresses = ipAddresses; - this.wireguardPubkey = wireguardPubkey; - this.wireguardKeyTimestamp = wireguardKeyTimestamp; - } + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public Long wireguardKeyTimestamp; + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java index 3d0d052a877..35ca757ebbe 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java @@ -92,6 +92,10 @@ public class NodeRepositoryNode { @JsonProperty("trustStore") @JsonInclude(JsonInclude.Include.NON_EMPTY) public List trustStore; + @JsonProperty("wireguard") + public WireguardKeyWithTimestamp wireguardKeyWithTimestamp; + + // TODO wg: remove separate key and timestamp when all nodes use new keyWithTimestamp @JsonProperty("wireguardPubkey") @JsonInclude(JsonInclude.Include.NON_EMPTY) public String wireguardPubkey; @@ -141,12 +145,24 @@ public class NodeRepositoryNode { ", exclusiveTo='" + exclusiveTo + '\'' + ", history=" + history + ", trustStore=" + trustStore + - ", wireguardPubkey=" + wireguardPubkey + - ", wireguardKeyTimestamp=" + wireguardKeyTimestamp + + ", wireguard=" + wireguardKeyTimestamp + ", reports=" + reports + '}'; } + @JsonIgnoreProperties(ignoreUnknown = true) + public static class WireguardKeyWithTimestamp { + @JsonProperty("key") + public String key; + @JsonProperty("timestamp") + public long timestamp; + + public WireguardKeyWithTimestamp(@JsonProperty("key") String key, @JsonProperty("timestamp") long timestamp) { + this.key = key; + this.timestamp = timestamp; + } + } + @JsonIgnoreProperties(ignoreUnknown = true) public static class Owner { @JsonProperty("tenant") diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java index b5428f57f08..e5ab9a1ce31 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java @@ -1,10 +1,9 @@ package com.yahoo.vespa.hosted.node.admin.wireguard; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import java.time.Instant; import java.util.List; /** @@ -15,8 +14,7 @@ import java.util.List; */ public record WireguardPeer(HostName hostname, List ipAddresses, - WireguardKey publicKey, - Instant wireguardKeyTimestamp) implements Comparable { + WireguardKeyWithTimestamp keyWithTimestamp) implements Comparable { public WireguardPeer { if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for peer node " + hostname.value()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 98e65d03f2f..ee3eac22d02 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; @@ -140,6 +141,7 @@ public class RealNodeRepositoryTest { var dockerImage = "registry.example.com/repo/image-1:6.2.3"; var wireguardKey = WireguardKey.from("111122223333444455556666777788889999000042c="); var wireguardKeyTimestamp = Instant.ofEpochMilli(123L); // Instant from clock in MockNodeRepository + var keyWithTimestamp = new WireguardKeyWithTimestamp(wireguardKey, wireguardKeyTimestamp); nodeRepositoryApi.updateNodeAttributes( hostname, @@ -151,8 +153,7 @@ public class RealNodeRepositoryTest { NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode(hostname).orElseThrow(); assertEquals(1, hostSpec.currentRestartGeneration().orElseThrow()); assertEquals(dockerImage, hostSpec.currentDockerImage().orElseThrow().asString()); - assertEquals(wireguardKey.value(), hostSpec.wireguardPubkey().orElseThrow().value()); - assertEquals(wireguardKeyTimestamp, hostSpec.wireguardKeyTimestamp().orElseThrow()); + assertEquals(keyWithTimestamp, hostSpec.wireguardKeyWithTimestamp().orElseThrow()); } @Test @@ -215,7 +216,7 @@ public class RealNodeRepositoryTest { assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", "::201:1", "lololololololololololololololololololololoo=", - Instant.ofEpochMilli(456L)); + 456L); //// Exclave nodes //// @@ -227,16 +228,17 @@ public class RealNodeRepositoryTest { assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", "::101:1", "000011112222333344445555666677778888999900c=", - Instant.ofEpochMilli(123L)); + 123L); } private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, - String publicKey, Instant keyTimestamp) { + String publicKey, long keyTimestamp) { assertEquals(hostname, peer.hostname().value()); assertEquals(1, peer.ipAddresses().size()); assertIp(peer.ipAddresses().get(0), ipv6, 6); - assertEquals(publicKey, peer.publicKey().value()); - assertEquals(keyTimestamp, peer.wireguardKeyTimestamp()); + var expectedKeyWithTimestamp = new WireguardKeyWithTimestamp(WireguardKey.from(publicKey), + Instant.ofEpochMilli(keyTimestamp)); + assertEquals(expectedKeyWithTimestamp, peer.keyWithTimestamp()); } private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java index cd76b221c9e..6ee896e3db6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.wireguard; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; import org.junit.jupiter.api.Test; @@ -31,6 +32,7 @@ public class WireguardPeerTest { private static WireguardPeer peer(String hostname) { return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), - WireguardKey.generateRandomForTesting(), Instant.EPOCH); + new WireguardKeyWithTimestamp(WireguardKey.generateRandomForTesting(), Instant.EPOCH)); } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 24159b88a9b..d5e891a33c7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -64,8 +64,7 @@ public final class Node implements Nodelike { private final CloudAccount cloudAccount; /** Only set for configservers and exclave nodes */ - private final Optional wireguardPubKey; - private final Optional wireguardKeyTimestamp; + private final Optional wireguardPubKey; /** Record of the last event of each type happening to this node */ private final History history; @@ -96,8 +95,8 @@ public final class Node implements Nodelike { NodeType type, Reports reports, Optional modelName, Optional reservedTo, Optional exclusiveToApplicationId, Optional hostTTL, Optional hostEmptyAt, Optional exclusiveToClusterType, Optional switchHostname, - List trustStoreItems, CloudAccount cloudAccount, Optional wireguardPubKey, - Optional wireguardKeyTimestamp) { + List trustStoreItems, CloudAccount cloudAccount, + Optional wireguardPubKey) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.extraId = Objects.requireNonNull(extraId, "Extra ID cannot be null"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); @@ -120,7 +119,6 @@ public final class Node implements Nodelike { this.trustStoreItems = Objects.requireNonNull(trustStoreItems).stream().distinct().toList(); this.cloudAccount = Objects.requireNonNull(cloudAccount); this.wireguardPubKey = Objects.requireNonNull(wireguardPubKey); - this.wireguardKeyTimestamp = Objects.requireNonNull(wireguardKeyTimestamp); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -264,15 +262,10 @@ public final class Node implements Nodelike { } /** Returns the wireguard public key of this node. Only relevant for enclave nodes. */ - public Optional wireguardPubKey() { + public Optional wireguardPubKey() { return wireguardPubKey; } - /** Returns the timestamp of the wireguard key of this node. Only relevant for enclave nodes. */ - public Optional wireguardKeyTimestamp() { - return wireguardKeyTimestamp; - } - /** * Returns a copy of this where wantToFail is set to true and history is updated to reflect this. */ @@ -367,16 +360,14 @@ public final class Node implements Nodelike { public Node with(Status status) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a node with the flavor assigned to the given value */ @@ -385,40 +376,35 @@ public final class Node implements Nodelike { History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with given id set */ public Node withId(String id) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -460,24 +446,21 @@ public final class Node implements Nodelike { public Node with(Allocation allocation) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, extraId, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withReservedTo(TenantName tenant) { @@ -485,73 +468,59 @@ public final class Node implements Nodelike { throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, Optional.empty(), exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExtraId(Optional extraId) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostTTL(Duration hostTTL) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(hostTTL), hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withHostEmptyAt(Instant hostEmptyAt) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, Optional.ofNullable(hostEmptyAt), - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } - public Node withWireguardPubkey(WireguardKey wireguardPubkey) { + public Node withWireguardPubkey(WireguardKeyWithTimestamp wireguardPubkey) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, Optional.ofNullable(wireguardPubkey), - wireguardKeyTimestamp); - } - - public Node withWireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - Optional.ofNullable(wireguardKeyTimestamp)); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, + Optional.ofNullable(wireguardPubkey)); } /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount, + wireguardPubKey); } /** Returns a copy of this node with switch hostname unset */ @@ -604,22 +573,19 @@ public final class Node implements Nodelike { public Node with(History history) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(Reports reports) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } public Node with(List trustStoreItems) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, hostTTL, hostEmptyAt, - exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey, - wireguardKeyTimestamp); + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } private static Optional requireNonEmptyString(Optional value, String message) { @@ -767,8 +733,7 @@ public final class Node implements Nodelike { private History history; private List trustStoreItems; private CloudAccount cloudAccount = CloudAccount.empty; - private WireguardKey wireguardPubKey; - private Instant wireguardKeyTimestamp; + private WireguardKeyWithTimestamp wireguardPubKey; private Builder(String id, String hostname, Flavor flavor, State state, NodeType type) { this.id = id; @@ -858,16 +823,11 @@ public final class Node implements Nodelike { return this; } - public Builder wireguardPubKey(WireguardKey wireguardPubKey) { + public Builder wireguardKey(WireguardKeyWithTimestamp wireguardPubKey) { this.wireguardPubKey = wireguardPubKey; return this; } - public Builder wireguardKeyTimestamp(Instant wireguardKeyTimestamp) { - this.wireguardKeyTimestamp = wireguardKeyTimestamp; - return this; - } - public Node build() { return new Node(id, Optional.empty(), Optional.ofNullable(ipConfig).orElse(IP.Config.EMPTY), hostname, Optional.ofNullable(parentHostname), flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), @@ -875,7 +835,7 @@ public final class Node implements Nodelike { Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), Optional.ofNullable(hostTTL), Optional.ofNullable(hostEmptyAt), Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), Optional.ofNullable(trustStoreItems).orElseGet(List::of), cloudAccount, - Optional.ofNullable(wireguardPubKey), Optional.ofNullable(wireguardKeyTimestamp)); + Optional.ofNullable(wireguardPubKey)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index 3c3868bfeb8..c7876032374 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -222,7 +222,7 @@ public class CuratorDb { node.type(), node.reports(), node.modelName(), node.reservedTo(), node.exclusiveToApplicationId(), node.hostTTL(), node.hostEmptyAt(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates(), - node.cloudAccount(), node.wireguardPubKey(), node.wireguardKeyTimestamp()); + node.cloudAccount(), node.wireguardPubKey()); curatorTransaction.add(createOrSet(nodePath(newNode), nodeSerializer.toJson(newNode))); writtenNodes.add(newNode); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 870e678a250..73531d650d5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.slime.ArrayTraverser; @@ -188,8 +189,10 @@ public class NodeSerializer { if (!node.cloudAccount().isUnspecified()) { object.setString(cloudAccountKey, node.cloudAccount().value()); } - node.wireguardPubKey().ifPresent(pubKey -> object.setString(wireguardPubKeyKey, pubKey.value())); - node.wireguardKeyTimestamp().ifPresent(timestamp -> object.setLong(wireguardKeyTimestampKey, timestamp.toEpochMilli())); + node.wireguardPubKey().ifPresent(pubKey -> { + object.setString(wireguardPubKeyKey, pubKey.key().value()); + object.setLong(wireguardKeyTimestampKey, pubKey.timestamp().toEpochMilli()); + }); } private void toSlime(Flavor flavor, Cursor object) { @@ -284,8 +287,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(switchHostnameKey)), trustedCertificatesFromSlime(object), SlimeUtils.optionalString(object.field(cloudAccountKey)).map(CloudAccount::from).orElse(CloudAccount.empty), - SlimeUtils.optionalString(object.field(wireguardPubKeyKey)).map(WireguardKey::from), - SlimeUtils.optionalInstant(object.field(wireguardKeyTimestampKey))); + wireguardKeyWithTimestampFromSlime(object.field(wireguardPubKeyKey), object.field(wireguardKeyTimestampKey))); } private Status statusFromSlime(Inspector object) { @@ -397,6 +399,13 @@ public class NodeSerializer { .toList(); } + private Optional wireguardKeyWithTimestampFromSlime(Inspector keyObject, Inspector timestampObject) { + if ( ! keyObject.valid()) return Optional.empty(); + return SlimeUtils.optionalString(keyObject).map( + key -> new WireguardKeyWithTimestamp(WireguardKey.from(key), + SlimeUtils.optionalInstant(timestampObject).orElse(null))); + } + // ----------------- Enum <-> string mappings ---------------------------------------- /** Returns the event type, or null if this event type should be ignored */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 9f1ab3dc3d5..cad034e01aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -108,7 +109,8 @@ public class NodePatcher { "reports", "trustStore", "vespaVersion", - "wireguardPubkey")); + "wireguardPubkey", // TODO wg: remove when all nodes use new key+timestamp format + "wireguard")); if (!disallowedFields.isEmpty()) { throw new IllegalArgumentException("Patching fields not supported: " + disallowedFields); } @@ -271,9 +273,13 @@ public class NodePatcher { return value.type() == Type.NIX ? node.withoutSwitchHostname() : node.withSwitchHostname(value.asString()); case "trustStore": return nodeWithTrustStore(node, value); - case "wireguardPubkey": - return node.withWireguardPubkey(SlimeUtils.optionalString(value).map(WireguardKey::new).orElse(null)) - .withWireguardKeyTimestamp(clock.instant()); + case "wireguard": + // This is where we set the key timestamp. + var key = SlimeUtils.optionalString(value.field("key")).map(WireguardKey::new).orElse(null); + return node.withWireguardPubkey(new WireguardKeyWithTimestamp(key, clock.instant())); + case "wireguardPubkey": // TODO wg: remove when all nodes use new key+timestamp format + var oldKey = SlimeUtils.optionalString(value).map(WireguardKey::new).orElse(null); + return node.withWireguardPubkey(new WireguardKeyWithTimestamp(oldKey, clock.instant())); default: throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index a8f526544d7..05bb0a27d69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; @@ -192,8 +193,13 @@ class NodesResponse extends SlimeJsonResponse { if (!node.cloudAccount().isUnspecified()) { object.setString("cloudAccount", node.cloudAccount().value()); } - node.wireguardPubKey().ifPresent(key -> object.setString("wireguardPubkey", key.value())); - node.wireguardKeyTimestamp().ifPresent(timestamp -> object.setLong("wireguardKeyTimestamp", timestamp.toEpochMilli())); + node.wireguardPubKey().ifPresent(key -> toSlime(key, object.setObject("wireguard"))); + + // TODO wg: remove when all nodes have upgraded to new key+timestamp format + node.wireguardPubKey().ifPresent(key -> { + object.setString("wireguardPubkey", key.key().value()); + object.setLong("wireguardKeyTimestamp", key.timestamp().toEpochMilli()); + }); } private Version resolveVersionFlag(StringFlag flag, Node node, Allocation allocation) { @@ -237,6 +243,11 @@ class NodesResponse extends SlimeJsonResponse { } } + static void toSlime(WireguardKeyWithTimestamp keyWithTimestamp, Cursor object) { + object.setString("key", keyWithTimestamp.key().value()); + object.setLong("timestamp", keyWithTimestamp.timestamp().toEpochMilli()); + } + private Optional currentContainerImage(Node node) { if (node.status().containerImage().isPresent()) { return node.status().containerImage(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java index 16e85dfa48a..e29c4f1b87a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; @@ -10,9 +10,9 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.IP; import java.net.InetAddress; -import java.time.Instant; import java.util.List; -import java.util.Optional; + +import static com.yahoo.vespa.hosted.provision.restapi.NodesResponse.toSlime; /** * A response containing the wireguard peer config for each configserver that has a public key. @@ -36,17 +36,20 @@ public class WireguardResponse extends SlimeJsonResponse { .toList(); if (ipAddresses.isEmpty()) continue; - addConfigserver(cfgArray.addObject(), cfg.hostname(), cfg.wireguardPubKey().get(), - cfg.wireguardKeyTimestamp(), ipAddresses); + addConfigserver(cfgArray.addObject(), cfg.hostname(), cfg.wireguardPubKey().get(), ipAddresses); } } - private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKey key, Optional keyTimestamp, + private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKeyWithTimestamp keyWithTimestamp, List ipAddresses) { cfgEntry.setString("hostname", hostname); - cfgEntry.setString("wireguardPubkey", key.value()); - cfgEntry.setLong("wireguardKeyTimestamp", keyTimestamp.orElse(Instant.EPOCH).toEpochMilli()); + + // TODO wg: remove when all nodes are using new key+timestamp format + cfgEntry.setString("wireguardPubkey", keyWithTimestamp.key().value()); + cfgEntry.setLong("wireguardKeyTimestamp", keyWithTimestamp.timestamp().toEpochMilli()); + NodesResponse.ipAddressesToSlime(ipAddresses, cfgEntry.setArray("ipAddresses")); + toSlime(keyWithTimestamp, cfgEntry.setObject("wireguard")); } private static boolean isPublicIp(String ipAddress) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 72225763381..2fb549acc11 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -21,6 +21,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.config.provision.WireguardKeyWithTimestamp; import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.ZoneEndpoint.AccessType; @@ -161,8 +162,8 @@ public class MockNodeRepository extends NodeRepository { // Emulate host in tenant account nodes.add(Node.create("dockerhost2", ipConfig(101, 1, 3), "dockerhost2.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host) - .wireguardPubKey(WireguardKey.from("000011112222333344445555666677778888999900c=")) - .wireguardKeyTimestamp(Instant.ofEpochMilli(123L)) + .wireguardKey(new WireguardKeyWithTimestamp(WireguardKey.from("000011112222333344445555666677778888999900c="), + Instant.ofEpochMilli(123L))) .cloudAccount(tenantAccount).build()); nodes.add(Node.create("dockerhost3", ipConfig(102, 1, 3), "dockerhost3.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host).cloudAccount(defaultCloudAccount).build()); @@ -176,8 +177,8 @@ public class MockNodeRepository extends NodeRepository { // Config servers nodes.add(Node.create("cfg1", ipConfig(201), "cfg1.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .cloudAccount(defaultCloudAccount) - .wireguardPubKey(WireguardKey.from("lololololololololololololololololololololoo=")) - .wireguardKeyTimestamp(Instant.ofEpochMilli(456L)) + .wireguardKey(new WireguardKeyWithTimestamp(WireguardKey.from("lololololololololololololololololololololoo="), + Instant.ofEpochMilli(456L))) .build()); nodes.add(Node.create("cfg2", ipConfig(202), "cfg2.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .cloudAccount(defaultCloudAccount) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json index 928e91861a2..54a0e7e9757 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json @@ -119,6 +119,10 @@ ], "additionalIpAddresses": [], "cloudAccount": "aws:111222333444", - "wireguardPubkey":"lololololololololololololololololololololoo=", + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 456 + }, + "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp": 456 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index 72b5483d849..d3f1a8082ae 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json @@ -117,6 +117,10 @@ "ipAddresses": ["127.0.101.1", "::101:1"], "additionalIpAddresses": ["::101:2", "::101:3", "::101:4"], "cloudAccount": "aws:777888999000", + "wireguard": { + "key": "000011112222333344445555666677778888999900c=", + "timestamp": 123 + }, "wireguardPubkey": "000011112222333344445555666677778888999900c=", "wireguardKeyTimestamp": 123 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json index d0d6df71fc1..404cf9a9a80 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json @@ -118,6 +118,10 @@ "ipAddresses": ["127.0.4.1", "::4:1"], "additionalIpAddresses": [], "cloudAccount": "aws:111222333444", + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 123 + }, "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp": 123 } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json index 7bee06adc87..8e9af7f680f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json @@ -4,7 +4,11 @@ "hostname": "cfg1.yahoo.com", "wireguardPubkey": "lololololololololololololololololololololoo=", "wireguardKeyTimestamp":456, - "ipAddresses": ["::201:1"] + "ipAddresses": ["::201:1"], + "wireguard": { + "key": "lololololololololololololololololololololoo=", + "timestamp": 456 + } } ] } -- cgit v1.2.3 From b65637ca31665103cff51bec49c69513905ea3e8 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 25 Sep 2023 10:51:16 +0200 Subject: Require larger heap Easier to later relax limit than to tighten --- .../vespa/model/application/validation/JvmHeapSizeValidator.java | 4 ++-- .../model/application/validation/JvmHeapSizeValidatorTest.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 2c5e0db14b9..9e231239521 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -28,7 +28,7 @@ public class JvmHeapSizeValidator extends Validator { } long jvmModelCost = appCluster.onnxModelCost().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { - int percentLimit = 10; + int percentLimit = 15; if (mp.percentage() < percentLimit) { throw new IllegalArgumentException( ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + @@ -36,7 +36,7 @@ public class JvmHeapSizeValidator extends Validator { "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") .formatted(clusterId, mp.percentage(), percentLimit, jvmModelCost / (1024D * 1024 * 1024))); } - double gbLimit = 0.4; + double gbLimit = 0.6; double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); if (availableMemoryGb < gbLimit) { throw new IllegalArgumentException( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 086f2fe778f..245887a5d03 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -34,16 +34,16 @@ class JvmHeapSizeValidatorTest { var deployState = createDeployState(8, 7L * 1024 * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); - String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 10%). Estimated cost of ONNX models is 7.00GB"; + String expectedMessage = "Allocated percentage of memory of JVM in cluster 'container' is too low (3% < 15%). Estimated cost of ONNX models is 7.00GB"; assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); } @Test void fails_on_too_low_heap_size() throws IOException, SAXException { - var deployState = createDeployState(2, 1024L * 1024 * 1024); + var deployState = createDeployState(2.2, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); var e = assertThrows(IllegalArgumentException.class, () -> new JvmHeapSizeValidator().validate(model, deployState)); - String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.30GB < 0.40GB). Estimated cost of ONNX models is 1.00GB."; + String expectedMessage = "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."; assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); } -- cgit v1.2.3 From 99e6c5335971b7c540313c00d72baf2cc1e2ec3e Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 25 Sep 2023 11:09:58 +0200 Subject: Return `JsonNode` --- .../main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index 7c86267c1b6..084f5dac94d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -29,6 +29,7 @@ import java.util.Map; public class OnnxModelProbe { private static final String binary = "vespa-analyze-onnx-model"; + private static final ObjectMapper jsonParser = new ObjectMapper(); static TensorType probeModel(ApplicationPackage app, Path modelPath, String outputName, Map inputTypes) { TensorType outputType = TensorType.empty; @@ -41,7 +42,7 @@ public class OnnxModelProbe { // Otherwise, run vespa-analyze-onnx-model if the model is available if (outputType.equals(TensorType.empty) && app.getFile(modelPath).exists()) { String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); - String jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); + var jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); outputType = outputTypeFromJson(jsonOutput, outputName); if ( ! outputType.equals(TensorType.empty)) { writeProbedOutputType(app, modelPath, contextKey, outputType); @@ -95,9 +96,7 @@ public class OnnxModelProbe { return TensorType.empty; } - private static TensorType outputTypeFromJson(String json, String outputName) throws IOException { - ObjectMapper m = new ObjectMapper(); - JsonNode root = m.readTree(json); + private static TensorType outputTypeFromJson(JsonNode root, String outputName) throws IOException { if ( ! root.isObject() || ! root.has("outputs")) { return TensorType.empty; } @@ -123,7 +122,7 @@ public class OnnxModelProbe { return out.toString(); } - private static String callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { + private static JsonNode callVespaAnalyzeOnnxModel(String jsonInput) throws IOException, InterruptedException { StringBuilder output = new StringBuilder(); ProcessBuilder processBuilder = new ProcessBuilder(binary, "--probe-types"); @@ -148,7 +147,7 @@ public class OnnxModelProbe { throw new IllegalArgumentException("Error from '" + binary + "'. Return code: " + returnCode + ". " + "Output: '" + output + "'"); } - return output.toString(); + return jsonParser.readTree(output.toString()); } } -- cgit v1.2.3 From bafecce9e84df5bb8e556490b7d961d623d364a3 Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Mon, 25 Sep 2023 11:21:37 +0200 Subject: Fix metric --- .../vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index 70eeb2b9f6c..ed383175cc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -69,7 +69,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { // Create metric for available certificates in the pool as a fraction of configured size int poolSize = certPoolSize.value(); long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count(); - metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? (available/poolSize) : 1.0), metric.createContext(Map.of())); + metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { provisionRandomizedCertificate(); -- cgit v1.2.3 From 1dc99c0e137ffe00f61225737184ece286d90cb5 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 25 Sep 2023 11:52:17 +0200 Subject: Probe memory usage when analyzing output types --- .../com/yahoo/vespa/model/ml/OnnxModelProbe.java | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index 084f5dac94d..38dda3e29ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -18,6 +18,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Optional; + +import static com.yahoo.yolean.Exceptions.uncheck; /** * Defers to 'vespa-analyze-onnx-model' to determine the output type given @@ -44,6 +47,7 @@ public class OnnxModelProbe { String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); var jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); outputType = outputTypeFromJson(jsonOutput, outputName); + writeMemoryStats(app, modelPath, MemoryStats.fromJson(jsonOutput)); if ( ! outputType.equals(TensorType.empty)) { writeProbedOutputType(app, modelPath, contextKey, outputType); } @@ -54,6 +58,22 @@ public class OnnxModelProbe { return outputType; } + public static Optional probeMemoryStats(ApplicationPackage app, Path modelPath) { + return Optional.of(app.getFile(memoryStatsPath(modelPath))) + .filter(ApplicationFile::exists) + .map(file -> MemoryStats.fromJson(uncheck(() -> jsonParser.readTree(file.createReader())))); + } + + private static void writeMemoryStats(ApplicationPackage app, Path modelPath, MemoryStats memoryStats) throws IOException { + String path = app.getFileReference(memoryStatsPath(modelPath)).getAbsolutePath(); + IOUtils.writeFile(path, memoryStats.toJson().toPrettyString(), false); + } + + private static Path memoryStatsPath(Path modelPath) { + var fileName = OnnxModelInfo.asValidIdentifier(modelPath.getRelative()) + ".memory_stats"; + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); + } + private static String createContextKey(String onnxName, Map inputTypes) { StringBuilder key = new StringBuilder().append(onnxName).append(":"); inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) @@ -150,4 +170,13 @@ public class OnnxModelProbe { return jsonParser.readTree(output.toString()); } + public record MemoryStats(long vmSize, long vmRss) { + static MemoryStats fromJson(JsonNode json) { + return new MemoryStats(json.get("vm_size").asLong(), json.get("vm_rss").asLong()); + } + JsonNode toJson() { + return jsonParser.createObjectNode().put("vm_size", vmSize).put("vm_rss", vmRss); + } + } + } -- cgit v1.2.3 From 89c5d77473fa8c319842d7afbbf69b92fb91268a Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 25 Sep 2023 13:04:29 +0200 Subject: Track user configured urls separately --- .../application/validation/UrlConfigValidator.java | 9 +++------ .../model/container/ApplicationContainerCluster.java | 20 +++++++++++++++++++- .../model/filedistribution/UserConfiguredFiles.java | 9 +++++++-- .../filedistribution/UserConfiguredFilesTest.java | 3 ++- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java index e6cd1a9e192..8332ba0387a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java @@ -5,8 +5,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import java.util.Optional; - /** * Validates that config using s3:// urls is used in public system and with nodes that are exclusive. * @@ -33,7 +31,7 @@ public class UrlConfigValidator extends Validator { } private static void validateS3UlsInConfig(DeployState state, ApplicationContainerCluster cluster, boolean isExclusive) { - if (hasUrlInConfig(state)) { + if (hasUrlInConfig(cluster)) { // TODO: Would be even better if we could add which config/field the url is set for in the error message String message = "Found s3:// urls in config for container cluster " + cluster.getName(); if ( ! state.zone().system().isPublic()) @@ -44,9 +42,8 @@ public class UrlConfigValidator extends Validator { } } - private static boolean hasUrlInConfig(DeployState state) { - return state.getFileRegistry().export().stream() - .anyMatch(fileReference -> fileReference.relativePath.startsWith("s3://")); + private static boolean hasUrlInConfig(ApplicationContainerCluster cluster) { + return cluster.userConfiguredUrls().all().size() > 0; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index b9021912244..eb2b39f74e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -40,9 +40,11 @@ import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.filedistribution.UserConfiguredFiles; +import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -100,6 +102,8 @@ public final class ApplicationContainerCluster extends ContainerCluster endpoints = List.of(); + private UserConfiguredUrls userConfiguredUrls = new UserConfiguredUrls(); + public ApplicationContainerCluster(TreeConfigProducer parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); this.tlsClientAuthority = deployState.tlsClientAuthority(); @@ -127,6 +131,8 @@ public final class ApplicationContainerCluster extends ContainerCluster component : getAllComponents()) { files.register(component); } @@ -390,4 +398,14 @@ public final class ApplicationContainerCluster extends ContainerCluster urls = new HashSet<>(); + + public void add(String url) { urls.add(url); } + + public Set all() { return urls; } + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 8d8b4d72f4d..03541ecadf3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -11,6 +11,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; + import com.yahoo.yolean.Exceptions; import java.io.File; @@ -21,6 +22,8 @@ import java.util.Map; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; + /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. * @@ -30,10 +33,12 @@ public class UserConfiguredFiles implements Serializable { private final FileRegistry fileRegistry; private final DeployLogger logger; + private final UserConfiguredUrls userConfiguredUrls; - public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger) { + public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, UserConfiguredUrls userConfiguredUrls) { this.fileRegistry = fileRegistry; this.logger = logger; + this.userConfiguredUrls = userConfiguredUrls; } /** @@ -134,7 +139,7 @@ public class UserConfiguredFiles implements Serializable { if (isModelType) { var modelReference = ModelReference.valueOf(builder.getValue()); if (modelReference.path().isEmpty()) { - modelReference.url().ifPresent(url -> fileRegistry.addUri(url.value())); + modelReference.url().ifPresent(url -> userConfiguredUrls.add(url.value())); return; } path = Path.fromString(modelReference.path().get().value()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index bb5ba840c2c..e2a25642575 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -68,7 +69,7 @@ public class UserConfiguredFilesTest { } private UserConfiguredFiles userConfiguredFiles() { - return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger()); + return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger(), new ApplicationContainerCluster.UserConfiguredUrls()); } @BeforeEach -- cgit v1.2.3 From a4d231f899a77dec7f390c5c0e9b60cb3ea3e866 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 25 Sep 2023 13:32:22 +0200 Subject: Check only for s3 urls in validator --- .../vespa/model/application/validation/UrlConfigValidator.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java index 8332ba0387a..d9dd3729bd3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UrlConfigValidator.java @@ -31,7 +31,7 @@ public class UrlConfigValidator extends Validator { } private static void validateS3UlsInConfig(DeployState state, ApplicationContainerCluster cluster, boolean isExclusive) { - if (hasUrlInConfig(cluster)) { + if (hasS3UrlInConfig(cluster)) { // TODO: Would be even better if we could add which config/field the url is set for in the error message String message = "Found s3:// urls in config for container cluster " + cluster.getName(); if ( ! state.zone().system().isPublic()) @@ -42,8 +42,9 @@ public class UrlConfigValidator extends Validator { } } - private static boolean hasUrlInConfig(ApplicationContainerCluster cluster) { - return cluster.userConfiguredUrls().all().size() > 0; + private static boolean hasS3UrlInConfig(ApplicationContainerCluster cluster) { + return cluster.userConfiguredUrls().all().stream() + .anyMatch(url -> url.startsWith("s3://")); } } -- cgit v1.2.3 From 2a537e9ce9223110ca2bbedd7e88139c24524049 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Mon, 25 Sep 2023 13:33:12 +0200 Subject: Use memory statistics from model probing in calculation --- .../com/yahoo/config/model/api/OnnxModelCost.java | 5 ++-- .../yahoo/vespa/model/DefaultOnnxModelCost.java | 29 +++++++++++++++++++--- .../container/ApplicationContainerCluster.java | 3 ++- .../validation/JvmHeapSizeValidatorTest.java | 3 ++- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index 422ceba8074..595cd97e6b6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -4,6 +4,7 @@ package com.yahoo.config.model.api; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; /** @@ -11,7 +12,7 @@ import com.yahoo.config.application.api.DeployLogger; */ public interface OnnxModelCost { - Calculator newCalculator(DeployLogger logger); + Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger); interface Calculator { long aggregatedModelCostInBytes(); @@ -20,7 +21,7 @@ public interface OnnxModelCost { } static OnnxModelCost disabled() { - return (__) -> new Calculator() { + return (__, ___) -> new Calculator() { @Override public long aggregatedModelCostInBytes() { return 0; } @Override public void registerModel(ApplicationFile path) {} @Override public void registerModel(ModelReference ref) {} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java index 76733872882..9794cfe4ad7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.model; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.vespa.model.ml.OnnxModelProbe; import java.io.IOException; import java.net.URI; @@ -29,16 +31,18 @@ import static com.yahoo.yolean.Exceptions.uncheck; public class DefaultOnnxModelCost implements OnnxModelCost { @Override - public Calculator newCalculator(DeployLogger logger) { - return new CalculatorImpl(logger); + public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { + return new CalculatorImpl(appPkg, logger); } private static class CalculatorImpl implements Calculator { private final DeployLogger log; + private final ApplicationPackage appPkg; private final ConcurrentMap modelCost = new ConcurrentHashMap<>(); - private CalculatorImpl(DeployLogger log) { + private CalculatorImpl(ApplicationPackage appPkg, DeployLogger log) { + this.appPkg = appPkg; this.log = log; } @@ -52,7 +56,17 @@ public class DefaultOnnxModelCost implements OnnxModelCost { String path = f.getPath().getRelative(); if (alreadyAnalyzed(path)) return; log.log(Level.FINE, () -> "Register model '%s'".formatted(path)); - deductJvmHeapSizeWithModelCost(f.exists() ? f.getSize() : 0, path); + if (f.exists()) { + var memoryStats = OnnxModelProbe.probeMemoryStats(appPkg, f.getPath()).orElse(null); + if (memoryStats != null) { + log.log(Level.FINE, () -> "Register model '%s' with memory stats: %s".formatted(path, memoryStats)); + deductJvmHeapSizeWithModelCost(f.getSize(), memoryStats, path); + } else { + deductJvmHeapSizeWithModelCost(f.getSize(), path); + } + } else { + deductJvmHeapSizeWithModelCost(0, path); + } } @Override @@ -92,6 +106,13 @@ public class DefaultOnnxModelCost implements OnnxModelCost { modelCost.put(source, estimatedCost); } + private void deductJvmHeapSizeWithModelCost(long size, OnnxModelProbe.MemoryStats stats, String source) { + long estimatedCost = (long)(1.1D * stats.vmSize()); + log.log(Level.FINE, () -> + "Estimated %s footprint for model of size %s ('%s')".formatted(mb(estimatedCost), mb(size), source)); + modelCost.put(source, estimatedCost); + } + private boolean alreadyAnalyzed(String source) { return modelCost.containsKey(source); } private static String mb(long bytes) { return "%dMB".formatted(bytes / (1024*1024)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index da6e3387d6a..d49276457b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -130,7 +130,8 @@ public final class ApplicationContainerCluster extends ContainerCluster 0 ? Math.min(99, deployState.featureFlags().heapSizePercentage()) : defaultHeapSizePercentageOfAvailableMemory; - onnxModelCost = deployState.onnxModelCost().newCalculator(deployState.getDeployLogger()); + onnxModelCost = deployState.onnxModelCost().newCalculator( + deployState.getApplicationPackage(), deployState.getDeployLogger()); logger = deployState.getDeployLogger(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 086f2fe778f..9ce1bb0d02a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.OnnxModelCost; @@ -112,7 +113,7 @@ class JvmHeapSizeValidatorTest { ModelCostDummy(long modelCost) { this.modelCost = modelCost; } - @Override public Calculator newCalculator(DeployLogger logger) { return this; } + @Override public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { return this; } @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } @Override public void registerModel(ApplicationFile path) {} -- cgit v1.2.3 From 8c31125738e95d9c09d49e3672a487a0280d88d6 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 25 Sep 2023 14:02:10 +0200 Subject: Revert "Remove unused endpoint name building from config-model" --- .../model/api/ApplicationClusterEndpoint.java | 75 +++++++++++++++++++++- .../container/ApplicationContainerCluster.java | 60 ++++++++++++----- .../model/container/ContainerClusterTest.java | 59 ++++++++++++++--- .../server/model/LbServicesProducerTest.java | 5 +- 4 files changed, 167 insertions(+), 32 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 215439aa42a..69749ee6f96 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -2,8 +2,15 @@ package com.yahoo.config.model.api; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; + import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Represents one endpoint for an application cluster @@ -147,21 +154,83 @@ public class ApplicationClusterEndpoint { } - public record DnsName(String name) implements Comparable { + public static class DnsName implements Comparable { + + private static final int MAX_LABEL_LENGTH = 63; + + private final String name; + + private DnsName(String name) { + this.name = name; + } public String value() { return name; } + public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { + String name = dnsParts(systemName, cluster, applicationId) + .filter(Objects::nonNull) // remove null values that were "default" + .collect(Collectors.joining("--")); + return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label + } + + public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { + String name = dnsParts(systemName, cluster, applicationId) + .filter(Objects::nonNull) // remove null values that were "default" + .map(DnsName::sanitize) + .collect(Collectors.joining(".")); + return new DnsName(name + suffix); + } + public static DnsName from(String name) { return new DnsName(name); } + private static Stream dnsParts(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId) { + return Stream.of( + nullIfDefault(cluster.value()), + systemPart(systemName), + nullIfDefault(applicationId.instance().value()), + applicationId.application().value(), + applicationId.tenant().value() + ); + } + + /** + * Remove any invalid characters from the hostnames + */ + private static String sanitize(String id) { + return shortenIfNeeded(id.toLowerCase() + .replace('_', '-') + .replaceAll("[^a-z0-9-]*", "")); + } + + /** + * Truncate the given string at the front so its length does not exceed 63 characters. + */ + private static String shortenIfNeeded(String id) { + return id.substring(Math.max(0, id.length() - MAX_LABEL_LENGTH)); + } + + private static String nullIfDefault(String string) { + return Optional.of(string).filter(s -> !s.equals("default")).orElse(null); + } + + private static String systemPart(SystemName systemName) { + return "cd".equals(systemName.value()) ? systemName.value() : null; + } + + @Override + public String toString() { + return "DnsName{" + + "name='" + name + '\'' + + '}'; + } + @Override public int compareTo(DnsName o) { return name.compareTo(o.name); } - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 0945dfaf54a..2227831a8a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -11,11 +11,13 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -224,23 +226,49 @@ public final class ApplicationContainerCluster extends ContainerCluster hosts = getContainers().stream().map(AbstractService::getHostName).sorted().toList(); List endpoints = new ArrayList<>(); - deployState.getEndpoints().stream() - .filter(ce -> ce.clusterId().equals(getName())) - .forEach(ce -> ce.names().forEach( - name -> endpoints.add(ApplicationClusterEndpoint.builder() - .scope(ce.scope()) - .weight(ce.weight().orElse(1)) - .routingMethod(ce.routingMethod()) - .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ce.authMethod()) - .build()) - )); - this.endpoints = Collections.unmodifiableList(endpoints); + + List hosts = getContainers().stream() + .map(AbstractService::getHostName) + .sorted() + .toList(); + + Set endpointsFromController = deployState.getEndpoints(); + // Add zone-scoped endpoints if not provided by the controller + // TODO(mpolden): Remove this when controller always includes zone-scope endpoints, and config models < 8.230 are gone + if (endpointsFromController.stream().noneMatch(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.zone)) { + for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { + ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( + deployState.zone().system(), + ClusterSpec.Id.from(getName()), + deployState.getProperties().applicationId(), + suffix); + endpoints.add(ApplicationClusterEndpoint.builder() + .zoneScope() + .sharedL4Routing() + .dnsName(l4Name) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ApplicationClusterEndpoint.AuthMethod.mtls) + .build()); + } + } + + // Include all endpoints provided by controller + endpointsFromController.stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .weight(ce.weight().orElse(1)) // Default to weight=1 if not set + .routingMethod(ce.routingMethod()) + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ce.authMethod()) + .build()) + )); + this.endpoints = List.copyOf(endpoints); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 3bdb60a0a8d..894fc55c014 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -42,14 +42,14 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; +import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * @author Simon Thoresen Hult @@ -365,23 +365,62 @@ public class ContainerClusterTest { @Test void generatesCorrectRoutingInfo() { + // main system: assertNames(main, ApplicationId.from("t1", "a1", "i1"), - Set.of(new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.i1.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), + Set.of(), List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar")), - new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), + Set.of(), + List.of("search-cluster.a1.t1.endpoint.suffix")); + + assertNames(main, + ApplicationId.from("t1", "default", "default"), + Set.of(), + List.of("search-cluster.default.t1.endpoint.suffix")); + + assertNames(main, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), - new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); + + // cd system: + assertNames(cd, + ApplicationId.from("t1", "a1", "i1"), + Set.of(), + List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); + + assertNames(cd, + ApplicationId.from("t1", "a1", "default"), + Set.of(), + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); + + assertNames(cd, + ApplicationId.from("t1", "default", "default"), + Set.of(), + List.of("search-cluster.cd.default.t1.endpoint.suffix")); + + assertNames(cd, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + List.of("search-cluster.cd.a1.t1.endpoint.suffix")); + + assertNames(cd, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), + List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); + } private void assertNames(SystemName systemName, ApplicationId appId, Set globalEndpoints, List expectedSharedL4Names) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 8d1b0fefbaf..ca2f9da3273 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -50,11 +50,10 @@ import static org.junit.Assert.assertTrue; public class LbServicesProducerTest { private static final Set endpoints = Set.of( - new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint1.suffix")), - new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint2.suffix")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); + private static final List zoneDnsSuffixes = List.of(".endpoint1.suffix", ".endpoint2.suffix"); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -229,7 +228,7 @@ public class LbServicesProducerTest { private TestProperties getTestproperties(ApplicationId applicationId) { return new TestProperties() .setHostedVespa(true) + .setZoneDnsSuffixes(zoneDnsSuffixes) .setApplicationId(applicationId); } - } -- cgit v1.2.3 From 512c1a8012ae0a684b3e84f3090ffa9a140769e5 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 21 Sep 2023 15:39:47 +0200 Subject: Remove fallback to zone-scoped endpoint building --- .../model/api/ApplicationClusterEndpoint.java | 8 +-- .../container/ApplicationContainerCluster.java | 60 ++++++---------------- .../model/container/ContainerClusterTest.java | 59 ++++----------------- .../server/model/LbServicesProducerTest.java | 5 +- 4 files changed, 30 insertions(+), 102 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 69749ee6f96..0276985d6a6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -168,13 +168,7 @@ public class ApplicationClusterEndpoint { return name; } - public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(systemName, cluster, applicationId) - .filter(Objects::nonNull) // remove null values that were "default" - .collect(Collectors.joining("--")); - return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label - } - + // TODO(mpolden): Remove when config-models < 8.232 are gone public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 2227831a8a0..0945dfaf54a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -11,13 +11,11 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ApplicationClusterInfo; -import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.OnnxModelCost; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -226,49 +224,23 @@ public final class ApplicationContainerCluster extends ContainerCluster hosts = getContainers().stream().map(AbstractService::getHostName).sorted().toList(); List endpoints = new ArrayList<>(); - - List hosts = getContainers().stream() - .map(AbstractService::getHostName) - .sorted() - .toList(); - - Set endpointsFromController = deployState.getEndpoints(); - // Add zone-scoped endpoints if not provided by the controller - // TODO(mpolden): Remove this when controller always includes zone-scope endpoints, and config models < 8.230 are gone - if (endpointsFromController.stream().noneMatch(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.zone)) { - for (String suffix : deployState.getProperties().zoneDnsSuffixes()) { - ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( - deployState.zone().system(), - ClusterSpec.Id.from(getName()), - deployState.getProperties().applicationId(), - suffix); - endpoints.add(ApplicationClusterEndpoint.builder() - .zoneScope() - .sharedL4Routing() - .dnsName(l4Name) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ApplicationClusterEndpoint.AuthMethod.mtls) - .build()); - } - } - - // Include all endpoints provided by controller - endpointsFromController.stream() - .filter(ce -> ce.clusterId().equals(getName())) - .forEach(ce -> ce.names().forEach( - name -> endpoints.add(ApplicationClusterEndpoint.builder() - .scope(ce.scope()) - .weight(ce.weight().orElse(1)) // Default to weight=1 if not set - .routingMethod(ce.routingMethod()) - .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) - .hosts(hosts) - .clusterId(getName()) - .authMethod(ce.authMethod()) - .build()) - )); - this.endpoints = List.copyOf(endpoints); + deployState.getEndpoints().stream() + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .weight(ce.weight().orElse(1)) + .routingMethod(ce.routingMethod()) + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ce.authMethod()) + .build()) + )); + this.endpoints = Collections.unmodifiableList(endpoints); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 894fc55c014..3bdb60a0a8d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -42,14 +42,14 @@ import java.util.Objects; import java.util.OptionalInt; import java.util.Set; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; -import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Simon Thoresen Hult @@ -365,62 +365,23 @@ public class ContainerClusterTest { @Test void generatesCorrectRoutingInfo() { - // main system: assertNames(main, ApplicationId.from("t1", "a1", "i1"), - Set.of(), + Set.of(new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.i1.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.i1.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix")); - - assertNames(main, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar")), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix")); assertNames(main, ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("search-cluster", zone, List.of("search-cluster.a1.t1.endpoint.suffix"), OptionalInt.empty(), sharedLayer4)), List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - - // cd system: - assertNames(cd, - ApplicationId.from("t1", "a1", "i1"), - Set.of(), - List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "default", "default"), - Set.of(), - List.of("search-cluster.cd.default.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), - List.of("search-cluster.cd.a1.t1.endpoint.suffix")); - - assertNames(cd, - ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), - new ContainerEndpoint("search-cluster", global, List.of("a.b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), - new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), - new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), - List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); - } private void assertNames(SystemName systemName, ApplicationId appId, Set globalEndpoints, List expectedSharedL4Names) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index ca2f9da3273..8d1b0fefbaf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -50,10 +50,11 @@ import static org.junit.Assert.assertTrue; public class LbServicesProducerTest { private static final Set endpoints = Set.of( + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint1.suffix")), + new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.zone, List.of("mydisc.foo.foo.endpoint2.suffix")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1", "rotation-2")), new ContainerEndpoint("mydisc", ApplicationClusterEndpoint.Scope.application, List.of("app-endpoint")) ); - private static final List zoneDnsSuffixes = List.of(".endpoint1.suffix", ".endpoint2.suffix"); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -228,7 +229,7 @@ public class LbServicesProducerTest { private TestProperties getTestproperties(ApplicationId applicationId) { return new TestProperties() .setHostedVespa(true) - .setZoneDnsSuffixes(zoneDnsSuffixes) .setApplicationId(applicationId); } + } -- cgit v1.2.3 From 493e1a9a82cac25727c99b75eb37acd62fc0f862 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 25 Sep 2023 14:36:45 +0200 Subject: Match tenant dimension --- .../main/java/com/yahoo/vespa/hosted/controller/RoutingController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 091836a1eea..27cd5e7e576 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -525,7 +525,8 @@ public class RoutingController { } public boolean generatedEndpointsEnabled(ApplicationId instance) { - return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()).value(); + return randomizedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()).value(); } private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { -- cgit v1.2.3 From 532e823b64a036e3894241507fd49b1ab79ccead Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Mon, 25 Sep 2023 15:51:11 +0200 Subject: Add another is_match member function to dfa fuzzy matcher that doesn't try to update directory iterator. --- .../dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp | 133 ++++++++++++++++++--- .../searchlib/attribute/dfa_fuzzy_matcher.cpp | 34 +++++- .../vespa/searchlib/attribute/dfa_fuzzy_matcher.h | 32 ++++- 3 files changed, 172 insertions(+), 27 deletions(-) diff --git a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp index 77e23a58163..c2a39779061 100644 --- a/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp +++ b/searchlib/src/tests/attribute/dfa_fuzzy_matcher/dfa_fuzzy_matcher_test.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -28,12 +29,22 @@ using vespalib::FuzzyMatcher; using vespalib::datastore::AtomicEntryRef; using vespalib::datastore::EntryRef; using vespalib::fuzzy::LevenshteinDfa; +using vespalib::Utf8Reader; +using vespalib::Utf8Writer; using StringEnumStore = EnumStoreT; using DictionaryEntry = std::pair; using RawDictionary = std::vector; using StringVector = std::vector; +namespace { + +const char* char_from_u8(const char8_t* p) { + return reinterpret_cast(p); +} + +} + RawDictionary read_dictionary() { @@ -110,11 +121,11 @@ struct MatchStats { template void -brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, MatchStats& stats, StringVector& matched_words) +brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - FuzzyMatcher matcher(target, 2, prefix_size, false); + FuzzyMatcher matcher(target, 2, prefix_size, cased); auto itr = view.begin(); size_t matches = 0; size_t seeks = 0; @@ -134,12 +145,18 @@ brute_force_fuzzy_match_in_dictionary(std::string_view target, const StringEnumS template void -dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, MatchStats& stats, StringVector& matched_words) +dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) { auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); vespalib::Timer timer; - DfaFuzzyMatcher matcher(target, 2, prefix_size, false, LevenshteinDfa::DfaType::Explicit); - std::string target_copy(target.substr(0, prefix_size)); + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + Utf8Reader reader(vespalib::stringref(target.data(), target.size())); + std::string target_copy; + Utf8Writer writer(target_copy); + for (size_t pos = 0; pos < prefix_size && reader.hasMore(); ++pos) { + auto code_point = reader.getChar(); + writer.putChar(code_point); + } auto prefix_cmp = store.make_folded_comparator_prefix(target_copy.c_str()); auto itr = prefix_size > 0 ? view.lowerBound(AtomicEntryRef(), prefix_cmp) : view.begin(); auto itr_end = itr; @@ -169,10 +186,58 @@ dfa_fuzzy_match_in_dictionary(std::string_view target, const StringEnumStore& st stats.add_sample(matches, seeks, timer.elapsed()); } -struct DfaFuzzyMatcherTest : public ::testing::Test { +template +void +dfa_fuzzy_match_in_dictionary_no_skip(std::string_view target, const StringEnumStore& store, uint32_t prefix_size, bool cased, MatchStats& stats, StringVector& matched_words) +{ + auto view = store.get_dictionary().get_posting_dictionary().getFrozenView(); + vespalib::Timer timer; + DfaFuzzyMatcher matcher(target, 2, prefix_size, cased, LevenshteinDfa::DfaType::Explicit); + auto itr = view.begin(); + size_t matches = 0; + size_t seeks = 0; + for (;itr.valid(); ++itr) { + auto word = store.get_value(itr.getKey().load_relaxed()); + if (matcher.is_match(word)) { + ++matches; + if (collect_matches) { + matched_words.push_back(word); + } + } else { + ++seeks; + } + } + stats.add_sample(matches, seeks, timer.elapsed()); +} + +struct TestParam +{ + vespalib::string _name; + bool _cased; + + TestParam(vespalib::string name, bool cased) + : _name(std::move(name)), + _cased(cased) + { + } + TestParam(const TestParam&); + ~TestParam(); +}; + +TestParam::TestParam(const TestParam&) = default; + +TestParam::~TestParam() = default; + +std::ostream& operator<<(std::ostream& os, const TestParam& param) +{ + os << param._name; + return os; +} + +struct DfaFuzzyMatcherTest : public ::testing::TestWithParam { StringEnumStore store; DfaFuzzyMatcherTest() - : store(true, DictionaryConfig(DictionaryConfig::Type::BTREE, DictionaryConfig::Match::UNCASED)) + : store(true, DictionaryConfig(DictionaryConfig::Type::BTREE, GetParam()._cased ? DictionaryConfig::Match::CASED : DictionaryConfig::Match::UNCASED)) {} void populate_dictionary(const StringVector& words) { auto updater = store.make_batch_updater(); @@ -187,18 +252,27 @@ struct DfaFuzzyMatcherTest : public ::testing::Test { MatchStats stats; StringVector brute_force_matches; StringVector dfa_matches; + StringVector dfa_no_skip_matches; + bool cased = GetParam()._cased; SCOPED_TRACE(target); - brute_force_fuzzy_match_in_dictionary(target, store, prefix_size, stats, brute_force_matches); - dfa_fuzzy_match_in_dictionary(target, store, prefix_size, stats, dfa_matches); + brute_force_fuzzy_match_in_dictionary(target, store, prefix_size, cased, stats, brute_force_matches); + dfa_fuzzy_match_in_dictionary(target, store, prefix_size, cased, stats, dfa_matches); + dfa_fuzzy_match_in_dictionary_no_skip(target, store, prefix_size, cased, stats, dfa_no_skip_matches); EXPECT_EQ(exp_matches, brute_force_matches); EXPECT_EQ(exp_matches, dfa_matches); + EXPECT_EQ(exp_matches, dfa_no_skip_matches); } void expect_matches(std::string_view target, const StringVector& exp_matches) { expect_prefix_matches(target, 0, exp_matches); } }; -TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) +INSTANTIATE_TEST_SUITE_P(DfaFuzzyMatcherMultiTest, + DfaFuzzyMatcherTest, + testing::Values(TestParam("uncased", false), TestParam("cased", true)), + testing::PrintToStringParamName()); + +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) { StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", "for", "forbid", "force", "ford", "forearm", "forecast", "forest" }; @@ -211,10 +285,11 @@ TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary) expect_matches("forcecast", {"forecast"}); } -TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) +TEST_P(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) { + bool cased = GetParam()._cased; StringVector words = { "board", "boat", "bob", "door", "food", "foot", "football", "foothill", - "for", "forbid", "force", "ford", "forearm", "forecast", "forest" }; + "for", "forbid", "force", "ford", "forearm", "forecast", "forest", "H", "HA", "h", "ha", char_from_u8(u8"Ørn"), char_from_u8(u8"øre"), char_from_u8(u8"Ås"), char_from_u8(u8"ås")}; populate_dictionary(words); expect_prefix_matches("a", 1, {}); expect_prefix_matches("b", 1, {"bob"}); @@ -231,25 +306,46 @@ TEST_F(DfaFuzzyMatcherTest, fuzzy_match_in_dictionary_with_prefix_size) expect_prefix_matches("forcecast", 1, {"forecast"}); expect_prefix_matches("forcecast", 4, {}); expect_prefix_matches("z", 1, {}); + if (cased) { + expect_prefix_matches("h", 1, {"h", "ha"}); + expect_prefix_matches(char_from_u8(u8"Ø"), 1, {char_from_u8(u8"Ørn")}); + expect_prefix_matches(char_from_u8(u8"ø"), 1, {char_from_u8(u8"øre")}); + expect_prefix_matches(char_from_u8(u8"å"), 1, {char_from_u8(u8"ås")}); + /* Corner case: prefix length > target length means exact match */ + expect_prefix_matches("h", 2, {"h"}); + } else { + expect_prefix_matches("h", 1, {"H", "h", "HA", "ha"}); + expect_prefix_matches(char_from_u8(u8"ø"), 1, {char_from_u8(u8"øre"), char_from_u8(u8"Ørn")}); + expect_prefix_matches(char_from_u8(u8"å"), 1, {char_from_u8(u8"Ås"), char_from_u8(u8"ås")}); + /* Corner case: prefix length > target length means exact match */ + expect_prefix_matches("h", 2, {"H", "h"}); + } } void -benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool dfa_algorithm) +benchmark_fuzzy_match_in_dictionary(const StringEnumStore& store, const RawDictionary& dict, size_t words_to_match, bool cased, bool dfa_algorithm) { MatchStats stats; StringVector dummy; for (size_t i = 0; i < std::min(words_to_match, dict.size()); ++i) { const auto& entry = dict[i]; if (dfa_algorithm) { - dfa_fuzzy_match_in_dictionary(entry.first, store, 0, stats, dummy); + dfa_fuzzy_match_in_dictionary(entry.first, store, 0, cased, stats, dummy); } else { - brute_force_fuzzy_match_in_dictionary(entry.first, store, 0, stats, dummy); + brute_force_fuzzy_match_in_dictionary(entry.first, store, 0, cased, stats, dummy); } } std::cout << (dfa_algorithm ? "DFA:" : "Brute force:") << " samples=" << stats.samples << ", avg_matches=" << stats.avg_matches() << ", avg_seeks=" << stats.avg_seeks() << ", avg_elapsed_ms=" << stats.avg_elapsed_ms() << std::endl; } -TEST_F(DfaFuzzyMatcherTest, benchmark_fuzzy_match_in_dictionary) +using DfaFuzzyMatcherBenchmarkTest = DfaFuzzyMatcherTest; + +INSTANTIATE_TEST_SUITE_P(DfaFuzzyMatcherBenchmarkMultiTest, + DfaFuzzyMatcherBenchmarkTest, + testing::Values(TestParam("uncased", false)), + testing::PrintToStringParamName()); + +TEST_P(DfaFuzzyMatcherBenchmarkTest, benchmark_fuzzy_match_in_dictionary) { if (!benchmarking_enabled()) { GTEST_SKIP() << "benchmarking not enabled"; @@ -258,8 +354,9 @@ TEST_F(DfaFuzzyMatcherTest, benchmark_fuzzy_match_in_dictionary) populate_dictionary(to_string_vector(dict)); std::cout << "Unique words: " << store.get_num_uniques() << std::endl; sort_by_freq(dict); - benchmark_fuzzy_match_in_dictionary(store, dict, dfa_words_to_match, true); - benchmark_fuzzy_match_in_dictionary(store, dict, brute_force_words_to_match, false); + bool cased = GetParam()._cased; + benchmark_fuzzy_match_in_dictionary(store, dict, dfa_words_to_match, cased, true); + benchmark_fuzzy_match_in_dictionary(store, dict, brute_force_words_to_match, cased, false); } int diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp index f66dd56c72f..b16fdc12a9a 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.cpp @@ -48,7 +48,8 @@ DfaFuzzyMatcher::DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uin _successor(), _prefix(extract_prefix(target, prefix_size, cased)), _successor_suffix(), - _prefix_size(prefix_size) + _prefix_size(prefix_size), + _cased(cased) { _successor = _prefix; } @@ -60,11 +61,38 @@ DfaFuzzyMatcher::skip_prefix(const char* word) const { Utf8ReaderForZTS reader(word); size_t pos = 0; - for (; pos < _prefix_size && reader.hasMore(); ++pos) { + for (; pos < _prefix.size() && reader.hasMore(); ++pos) { (void) reader.getChar(); } - assert(pos == _prefix_size); + assert(pos == _prefix.size()); return reader.get_current_ptr(); } +bool +DfaFuzzyMatcher::is_match(const char* word) const +{ + if (_prefix_size > 0) { + Utf8ReaderForZTS reader(word); + size_t pos = 0; + for (; pos < _prefix.size() && reader.hasMore(); ++pos) { + uint32_t code_point = reader.getChar(); + if (!_cased) { + code_point = LowerCase::convert(code_point); + } + if (code_point != _prefix[pos]) { + break; + } + } + if (!reader.hasMore() && pos == _prefix.size() && pos < _prefix_size) { + return true; + } + if (pos != _prefix_size) { + return false; + } + word = reader.get_current_ptr(); + } + auto match = _dfa.match(word); + return match.matches(); +} + } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h index 3d24948e044..7116b4d8662 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h @@ -5,6 +5,7 @@ #include "dfa_string_comparator.h" #include #include +#include namespace search::attribute { @@ -21,22 +22,41 @@ private: std::vector _prefix; std::vector _successor_suffix; uint32_t _prefix_size; + bool _cased; - const char*skip_prefix(const char* word) const; + const char* skip_prefix(const char* word) const; public: DfaFuzzyMatcher(std::string_view target, uint8_t max_edits, uint32_t prefix_size, bool cased, vespalib::fuzzy::LevenshteinDfa::DfaType dfa_type); ~DfaFuzzyMatcher(); + bool is_match(const char *word) const; + + /* + * If prefix size is nonzero then this variant of is_match() + * should only be called with words that starts with the extracted + * prefix of the target word. + * + * Caller must position iterator at right location using lower bound + * functionality in the dictionary. + */ template bool is_match(const char* word, DictionaryConstIteratorType& itr, const DfaStringComparator::DataStoreType& data_store) { if (_prefix_size > 0) { word = skip_prefix(word); - auto match = _dfa.match(word, _successor_suffix); - if (match.matches()) { - return true; + if (_prefix.size() < _prefix_size) { + if (*word == '\0') { + return true; + } + _successor.resize(_prefix.size()); + _successor.emplace_back(1); + } else { + auto match = _dfa.match(word, _successor_suffix); + if (match.matches()) { + return true; + } + _successor.resize(_prefix.size()); + _successor.insert(_successor.end(), _successor_suffix.begin(), _successor_suffix.end()); } - _successor.resize(_prefix.size()); - _successor.insert(_successor.end(), _successor_suffix.begin(), _successor_suffix.end()); } else { auto match = _dfa.match(word, _successor); if (match.matches()) { -- cgit v1.2.3 From 65515d8988ac49abccd1220ad8394f63693f1da1 Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Mon, 25 Sep 2023 16:02:36 +0200 Subject: Update last requested when fetching from pool --- .../vespa/hosted/controller/certificate/EndpointCertificates.java | 5 +++-- .../hosted/controller/certificate/EndpointCertificatesTest.java | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index e01da00a27e..d661fa189b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -140,10 +140,11 @@ public class EndpointCertificates { } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, candidate.get().certificate()), + EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); + curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), transaction); transaction.commit(); - return candidate.get().certificate(); + return certificate; } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 1cb43453918..a6d3b435dcb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -306,6 +306,9 @@ public class EndpointCertificatesTest { fail("Expected exception as certificate is not ready"); } catch (IllegalArgumentException ignored) {} + // Advance clock to verify last requested time + clock.advance(Duration.ofDays(3)); + // Certificate is assigned from pool instead. The previously assigned certificate will eventually be cleaned up // by EndpointCertificateMaintainer { // prod @@ -315,6 +318,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().randomizedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } { // dev @@ -325,6 +329,7 @@ public class EndpointCertificatesTest { assertEquals(certId, cert.get().randomizedId().get()); assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().randomizedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); + assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); } } -- cgit v1.2.3 From 8f010aec706f1c53b5d2d6775ddcbd61e234ae2e Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 25 Sep 2023 19:47:35 +0200 Subject: Try to add back method removed from API but still in use --- .../src/main/java/com/yahoo/config/model/api/OnnxModelCost.java | 1 + .../src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index 595cd97e6b6..9507f7db7b6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -12,6 +12,7 @@ import com.yahoo.config.application.api.DeployLogger; */ public interface OnnxModelCost { + default Calculator newCalculator(DeployLogger logger) { return newCalculator(null, logger); } Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger); interface Calculator { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java index 9794cfe4ad7..fddf8409376 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java @@ -56,7 +56,7 @@ public class DefaultOnnxModelCost implements OnnxModelCost { String path = f.getPath().getRelative(); if (alreadyAnalyzed(path)) return; log.log(Level.FINE, () -> "Register model '%s'".formatted(path)); - if (f.exists()) { + if (f.exists() && appPkg != null) { var memoryStats = OnnxModelProbe.probeMemoryStats(appPkg, f.getPath()).orElse(null); if (memoryStats != null) { log.log(Level.FINE, () -> "Register model '%s' with memory stats: %s".formatted(path, memoryStats)); -- cgit v1.2.3