diff options
8 files changed, 33 insertions, 8 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 446c32801e0..c3f83548a5b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -118,6 +118,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "log"; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index 595cd97e6b6..9507f7db7b6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -12,6 +12,7 @@ import com.yahoo.config.application.api.DeployLogger; */ public interface OnnxModelCost { + default Calculator newCalculator(DeployLogger logger) { return newCalculator(null, logger); } Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger); interface Calculator { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java index 9794cfe4ad7..fddf8409376 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/DefaultOnnxModelCost.java @@ -56,7 +56,7 @@ public class DefaultOnnxModelCost implements OnnxModelCost { String path = f.getPath().getRelative(); if (alreadyAnalyzed(path)) return; log.log(Level.FINE, () -> "Register model '%s'".formatted(path)); - if (f.exists()) { + if (f.exists() && appPkg != null) { var memoryStats = OnnxModelProbe.probeMemoryStats(appPkg, f.getPath()).orElse(null); if (memoryStats != null) { log.log(Level.FINE, () -> "Register model '%s' with memory stats: %s".formatted(path, memoryStats)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 4e97b20a3a9..ac679cc406c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -40,7 +40,6 @@ import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.filedistribution.UserConfiguredFiles; -import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -106,7 +105,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private List<ApplicationClusterEndpoint> endpoints = List.of(); - private UserConfiguredUrls userConfiguredUrls = new UserConfiguredUrls(); + private final UserConfiguredUrls userConfiguredUrls = new UserConfiguredUrls(); public ApplicationContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); @@ -163,6 +162,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat // Files referenced from user configs to all components. UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger(), + deployState.featureFlags(), userConfiguredUrls); for (Component<?, ?> component : getAllComponents()) { files.register(component); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 03541ecadf3..0095dec8079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -5,6 +5,7 @@ import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.path.Path; @@ -34,11 +35,15 @@ public class UserConfiguredFiles implements Serializable { private final FileRegistry fileRegistry; private final DeployLogger logger; private final UserConfiguredUrls userConfiguredUrls; + private final String unknownConfigDefinition; - public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, UserConfiguredUrls userConfiguredUrls) { + public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, + ModelContext.FeatureFlags featureFlags, + UserConfiguredUrls userConfiguredUrls) { this.fileRegistry = fileRegistry; this.logger = logger; this.userConfiguredUrls = userConfiguredUrls; + this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); } /** @@ -61,9 +66,12 @@ public class UserConfiguredFiles implements Serializable { private void register(ConfigPayloadBuilder builder, Map<Path, FileReference> registeredFiles, ConfigDefinitionKey key) { ConfigDefinition configDefinition = builder.getConfigDefinition(); if (configDefinition == null) { - // TODO: throw new IllegalArgumentException("Unable to find config definition for " + builder); - logger.logApplicationPackage(Level.INFO, "Unable to find config definition " + key + - ". Will not register files for file distribution for this config"); + String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config"; + switch (unknownConfigDefinition) { + case "log" -> logger.logApplicationPackage(Level.INFO, message); + case "warning" -> logger.logApplicationPackage(Level.WARNING, message); + case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key); + } return; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index e2a25642575..653bdbccf15 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.config.ConfigDefinition; @@ -69,7 +70,10 @@ public class UserConfiguredFilesTest { } private UserConfiguredFiles userConfiguredFiles() { - return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger(), new ApplicationContainerCluster.UserConfiguredUrls()); + return new UserConfiguredFiles(fileRegistry, + new BaseDeployLogger(), + new TestProperties(), + new ApplicationContainerCluster.UserConfiguredUrls()); } @BeforeEach diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 3e33b345437..7603ff61fa1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useReconfigurableDispatcher; private final int contentLayerMetadataFeatureLevel; private final boolean dynamicHeapSize; + private final String unknownConfigDefinition; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -245,6 +246,7 @@ public class ModelContextImpl implements ModelContext { this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER); this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL); this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE); + this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -296,6 +298,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } + @Override public String unknownConfigDefinition() { return unknownConfigDefinition; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e5b76bedecd..4baddf4944a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -62,6 +62,7 @@ public class Flags { " latency-amortized-over-requests, latency-amortized-over-time", "Takes effect at redeployment (requires restart)", INSTANCE_ID); + public static final UnboundStringFlag SUMMARY_DECODE_POLICY = defineStringFlag( "summary-decode-policy", "eager", List.of("baldersheim"), "2023-03-30", "2023-12-31", @@ -413,6 +414,13 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundStringFlag UNKNOWN_CONFIG_DEFINITION = defineStringFlag( + "unknonw-config-definition", "log", + List.of("hmusum"), "2023-09-25", "2024-11-01", + "How to handle user config referencing unknown config definitions. Valid values are log, warn, fail", + "Takes effect at redeployment", + INSTANCE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, |