diff options
28 files changed, 528 insertions, 150 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index 36d6efdf59b..d262c7bc862 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -27,21 +27,21 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Check whether or not this file is a directory. + * Checks whether this file is a directory. * * @return true if it is, false if not. */ public abstract boolean isDirectory(); /** - * Test whether or not this file exists. + * Tests whether this file exists. * * @return true if it exists, false if not. */ public abstract boolean exists(); /** - * Create a {@link Reader} for the contents of this file. + * Creates a {@link Reader} for the contents of this file. * * @return A {@link Reader} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -50,7 +50,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { /** - * Create an {@link InputStream} for the contents of this file. + * Creates an {@link InputStream} for the contents of this file. * * @return An {@link InputStream} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -58,7 +58,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract InputStream createInputStream() throws FileNotFoundException; /** - * Create a directory at the path represented by this file. Parent directories will + * Creates a directory at the path represented by this file. Parent directories will * be automatically created. * * @return this @@ -67,7 +67,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile createDirectory(); /** - * Write the contents from this reader to this file. Any existing content will be overwritten! + * Writes the contents from supplied reader to this file. Any existing content will be overwritten! * * @param input A reader pointing to the content that should be written. * @return this @@ -82,7 +82,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile appendFile(String value); /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @return a list of files in this directory. @@ -92,7 +92,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is a file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @param filter A filter functor for filtering path names @@ -101,7 +101,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract List<ApplicationFile> listFiles(PathFilter filter); /** - * List the files in this directory, optionally list files for subdirectories recursively as well. + * Lists the files in this directory, optionally lists files for subdirectories recursively as well. * * @param recurse Set to true if all files in the directory tree should be returned. * @return a list of files in this directory. @@ -121,7 +121,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Delete the file pointed to by this. If it is a non-empty directory, the operation will throw. + * Deletes the file pointed to by this. If this is a non-empty directory, the operation will throw. * * @return this. * @throws RuntimeException if the file is a directory and not empty. @@ -129,7 +129,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile delete(); /** - * Get the path that this file represents. + * Gets the path that this file represents. * * @return a Path */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9821f3b9568..f434d056bfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -166,7 +166,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger(), deployState.featureFlags(), - userConfiguredUrls); + userConfiguredUrls, + deployState.getApplicationPackage()); for (Component<?, ?> component : getAllComponents()) { files.register(component); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 0af970e016a..099255975b6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -64,8 +64,8 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { clientBindings.addAll(Arrays.asList(bindings)); } - public final Set<BindingPattern> getServerBindings() { - return Collections.unmodifiableSet(serverBindings); + public final Collection<BindingPattern> getServerBindings() { + return List.copyOf(serverBindings); } public final List<BindingPattern> getClientBindings() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index d2faff7850b..b14495756c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -9,6 +9,7 @@ import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.component.SimpleComponent; import java.util.ArrayList; @@ -24,13 +25,11 @@ import java.util.TreeSet; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster<?> cluster; - private volatile boolean isHostedVespa; private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); private final SortedSet<String> ignoredUserAgentsList = new TreeSet<>(); public JettyHttpServer(String componentId, ContainerCluster<?> cluster, DeployState deployState) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); - this.isHostedVespa = deployState.isHosted(); this.cluster = cluster; FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId); addChild(filterBindingsProviderComponent); @@ -42,8 +41,6 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro } } - public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } - public void addConnector(ConnectorFactory connectorFactory) { connectorFactories.add(connectorFactory); addChild(connectorFactory); @@ -64,10 +61,8 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .ignoredUserAgents(ignoredUserAgentsList) .searchHandlerPaths(List.of("/search")) ); - if (isHostedVespa) { - // Enable connection log hosted Vespa + if (cluster.getAllComponents().stream().anyMatch(c -> c instanceof ConnectionLogComponent)) builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); - } configureJettyThreadpool(builder); builder.stopTimeout(300); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7653d814d8a..119a3ad18c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -3,19 +3,14 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; -import static com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType.jsonAccessLog; - /** * Builds the config model for the standalone config server. * @@ -57,12 +52,6 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { } @Override - protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { - super.addHttp(deployState, spec, cluster, context); - cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); - } - - @Override protected void addModelEvaluationRuntime(ApplicationContainerCluster cluster) { // Model evaluation bundles are pre-installed in the standalone container. } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 47ae2f40414..a454c1141ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -24,6 +26,7 @@ import java.util.Optional; import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; +import static java.util.logging.Level.WARNING; /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. @@ -37,14 +40,17 @@ public class UserConfiguredFiles implements Serializable { private final DeployLogger logger; private final UserConfiguredUrls userConfiguredUrls; private final String unknownConfigDefinition; + private final ApplicationPackage applicationPackage; public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, ModelContext.FeatureFlags featureFlags, - UserConfiguredUrls userConfiguredUrls) { + UserConfiguredUrls userConfiguredUrls, + ApplicationPackage applicationPackage) { this.fileRegistry = fileRegistry; this.logger = logger; this.userConfiguredUrls = userConfiguredUrls; this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); + this.applicationPackage = applicationPackage; } /** @@ -69,7 +75,7 @@ public class UserConfiguredFiles implements Serializable { if (configDefinition == null) { String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config"; switch (unknownConfigDefinition) { - case "warning" -> logger.logApplicationPackage(Level.WARNING, message); + case "warning" -> logger.logApplicationPackage(WARNING, message); case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key); } return; @@ -155,9 +161,9 @@ public class UserConfiguredFiles implements Serializable { path = Path.fromString(builder.getValue()); } - File file = path.toFile(); - if (file.isDirectory() && (file.listFiles() == null || file.listFiles().length == 0)) - throw new IllegalArgumentException("Directory '" + path.getRelative() + "' is empty"); + ApplicationFile file = applicationPackage.getFile(path); + if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty())) + logger.logApplicationPackage(WARNING, "Directory '" + path.getRelative() + "' is empty"); FileReference reference = registeredFiles.get(path); if (reference == null) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f2e4ec052cb..a38a29893e0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -16,6 +16,7 @@ import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.container.logging.VespaAccessLog; +import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import org.junit.jupiter.api.Test; @@ -129,6 +130,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertEquals("default", config.cluster()); assertEquals(-1, config.queueSize()); assertEquals(256 * 1024, config.bufferSize()); + assertTrue(root.getConfig(ServerConfig.class, "default/container.0/DefaultHttpServer").connectionLog().enabled()); } @Test @@ -141,6 +143,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); Component<?, ?> fileConnectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); + assertFalse(root.getConfig(ServerConfig.class, "default/container.0/DefaultHttpServer").connectionLog().enabled()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java index 8a7ca27eec5..fdeea85c5a3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java @@ -1,11 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.usability.BindingsOverviewHandler; @@ -15,8 +15,10 @@ import com.yahoo.vespa.model.container.component.Handler; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.logging.Level; import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; @@ -25,7 +27,11 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasItem; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests for container model building with custom handlers. @@ -63,6 +69,31 @@ public class HandlerBuilderTest extends ContainerModelBuilderTestBase { } @Test + void warn_on_bindings_shared_by_multiple_handlers() { + class TestDeployLogger implements DeployLogger { + List<String> logs = new ArrayList<>(); + @Override public void log(Level level, String message) { logs.add(message); } + } + var clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <handler id='myHandler1'>", + " <binding>http://*/myhandler</binding>", + " <binding>https://*/myhandler</binding>", + " </handler>", + " <handler id='myHandler2'>", + " <binding>http://*/myhandler</binding>", + " <binding>https://*/myhandler</binding>", + " </handler>", + "</container>"); + var logger = new TestDeployLogger(); + createModel(root, logger, clusterElem); + assertEquals( + List.of("Binding 'http://*/myhandler' was already in use by handler 'myHandler1', but will now be taken over by handler: myHandler2", + "Binding 'https://*/myhandler' was already in use by handler 'myHandler1', but will now be taken over by handler: myHandler2"), + logger.logs); + } + + @Test void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index 523b0e74be1..b4a54548062 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -5,11 +5,15 @@ import com.yahoo.config.FileNode; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.schema.processing.ReservedRankingExpressionFunctionNamesTestCase; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -19,12 +23,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -69,12 +75,20 @@ public class UserConfiguredFilesTest { public String toString() { return export().toString(); } } - private UserConfiguredFiles userConfiguredFiles() { + return userConfiguredFiles(new MockApplicationPackage.Builder().build()); + } + + private UserConfiguredFiles userConfiguredFiles(ApplicationPackage applicationPackage) { + return userConfiguredFiles(applicationPackage, new BaseDeployLogger()); + } + + private UserConfiguredFiles userConfiguredFiles(ApplicationPackage applicationPackage, DeployLogger deployLogger) { return new UserConfiguredFiles(fileRegistry, - new BaseDeployLogger(), + deployLogger, new TestProperties(), - new ApplicationContainerCluster.UserConfiguredUrls()); + new ApplicationContainerCluster.UserConfiguredUrls(), + applicationPackage); } @BeforeEach @@ -289,18 +303,42 @@ public class UserConfiguredFilesTest { } @Test - void require_that_using_empty_dir_gives_sane_error_message(@TempDir Path tempDir) { - String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target")); + void require_that_using_empty_dir_fails(@TempDir Path tempDir) { + String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target") + 7); + ApplicationPackage applicationPackage = + new MockApplicationPackage.Builder() + .withRoot(tempDir.toFile().getParentFile()) + .withFiles(Map.of(com.yahoo.path.Path.fromString(tempDir.toFile().getAbsolutePath()), "")) + .build(); + + var logger = new TestDeployLogger(); + def.addPathDef("pathVal"); + builder.setField("pathVal", relativeTempDir); + fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); + userConfiguredFiles(applicationPackage, logger).register(producer); + assertEquals("Directory '" + relativeTempDir + "' is empty", logger.log); + } + + @Test + void require_that_using_non_existing_dir_fails() { + String relativeTempDir = "non-existing"; try { def.addPathDef("pathVal"); builder.setField("pathVal", relativeTempDir); - fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Invalid config in services.xml for 'mynamespace.myname': Directory '" + relativeTempDir + "' is empty", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': No such file or directory '" + relativeTempDir + "'", e.getMessage()); } } + private static class TestDeployLogger implements DeployLogger { + public String log = ""; + @Override + public void log(Level level, String message) { + log += message; + } + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index ed0f9aac884..d0b4ad9e917 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -2,8 +2,6 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import java.util.Objects; import java.util.Optional; @@ -79,6 +77,7 @@ public final class ClusterSpec { return combinedId; } + /** * Returns whether the physical hosts running the nodes of this application can * also run nodes of other applications. Using exclusive nodes for containers increases security and cost. @@ -96,12 +95,6 @@ public final class ClusterSpec { return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); } - // TODO: Remove after July 2023 - @Deprecated - public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); - } - /** Creates a ClusterSpec when requesting a cluster */ public static Builder request(Type type, Id id) { return new Builder(type, id); @@ -121,6 +114,7 @@ public final class ClusterSpec { private Optional<DockerImage> dockerImageRepo = Optional.empty(); private Version vespaVersion; private boolean exclusive = false; + private boolean provisionForApplication = false; private Optional<Id> combinedId = Optional.empty(); private ZoneEndpoint zoneEndpoint = ZoneEndpoint.defaultEndpoint; private boolean stateful; @@ -155,6 +149,11 @@ public final class ClusterSpec { return this; } + public Builder provisionForApplication(boolean provisionForApplication) { + this.provisionForApplication = provisionForApplication; + return this; + } + public Builder combinedId(Optional<Id> combinedId) { this.combinedId = combinedId; return this; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2680b4babb1..3de9d5aef4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -660,11 +660,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye log.log(Level.FINE, () -> "Remove unused file references last modified before " + instant); List<String> fileReferencesToDelete = sortedUnusedFileReferences(fileDirectory.getRoot(), fileReferencesInUse, instant); - if (fileReferencesToDelete.size() > 0) { - log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete); - fileReferencesToDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); + // Do max 20 at a time + var toDelete = fileReferencesToDelete.subList(0, Math.min(fileReferencesToDelete.size(), 20)); + if (toDelete.size() > 0) { + log.log(Level.FINE, () -> "Will delete file references not in use: " + toDelete); + toDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); + log.log(Level.FINE, () -> "Deleted " + toDelete.size() + " file references not in use"); } - return fileReferencesToDelete; + return toDelete; } private boolean isFileReferenceInUse(FileReference fileReference) { diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java index bb5a991c304..829d0c268e5 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -50,9 +50,7 @@ public class GlobalPhaseRanker { return Optional.empty(); } - public void rerankHits(Query query, Result result, String schema) { - var setup = globalPhaseSetupFor(query, schema).orElse(null); - if (setup == null) return; + static void rerankHitsImpl(GlobalPhaseSetup setup, Query query, Result result) { var mainSpec = setup.globalPhaseEvalSpec; var mainSrc = withQueryPrep(mainSpec.evalSource(), mainSpec.fromQuery(), query); int rerankCount = resolveRerankCount(setup, query); @@ -68,6 +66,13 @@ public class GlobalPhaseRanker { hideImplicitMatchFeatures(result, setup.matchFeaturesToHide); } + public void rerankHits(Query query, Result result, String schema) { + var setup = globalPhaseSetupFor(query, schema); + if (setup.isPresent()) { + rerankHitsImpl(setup.get(), query, result); + } + } + static Supplier<Evaluator> withQueryPrep(Supplier<Evaluator> evalSource, List<String> queryFeatures, Query query) { var prepared = PreparedInput.findFromQuery(query, queryFeatures); Supplier<Evaluator> supplier = () -> { @@ -80,7 +85,7 @@ public class GlobalPhaseRanker { return supplier; } - private void hideImplicitMatchFeatures(Result result, Collection<String> namesToHide) { + private static void hideImplicitMatchFeatures(Result result, Collection<String> namesToHide) { if (namesToHide.size() == 0) return; var filter = new MatchFeatureFilter(namesToHide); for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { @@ -94,7 +99,7 @@ public class GlobalPhaseRanker { if (newValue.fieldCount() == 0) { hit.removeField("matchfeatures"); } else { - hit.setField("matchfeatures", newValue); + hit.setField("matchfeatures", new FeatureData(newValue)); } } } @@ -106,7 +111,7 @@ public class GlobalPhaseRanker { .flatMap(evaluator -> evaluator.getGlobalPhaseSetup(query.getRanking().getProfile())); } - private int resolveRerankCount(GlobalPhaseSetup setup, Query query) { + private static int resolveRerankCount(GlobalPhaseSetup setup, Query query) { if (setup == null) { // there is no global-phase at all (ignore override) return 0; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java index 5ab2d7160f9..346acccd916 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java @@ -30,7 +30,7 @@ record PreparedInput(String name, Tensor value) { for (String queryFeatureName : queryFeatures) { String needed = "query(" + queryFeatureName + ")"; // searchers are recommended to place query features here: - var feature = rankFeatures.getTensor(queryFeatureName); + var feature = rankFeatures.getTensor(needed); if (feature.isPresent()) { result.add(new PreparedInput(needed, feature.get())); } else { @@ -38,6 +38,8 @@ record PreparedInput(String name, Tensor value) { var objList = rankProps.get(queryFeatureName); if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { result.add(new PreparedInput(needed, t)); + } else if (objList != null && objList.size() == 1 && objList.get(0) instanceof Double d) { + result.add(new PreparedInput(needed, Tensor.from(d))); } else { throw new IllegalArgumentException("missing query feature: " + queryFeatureName); } diff --git a/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java new file mode 100644 index 00000000000..ce9ac377908 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java @@ -0,0 +1,238 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Type; +import com.yahoo.data.access.helpers.MatchFeatureData; +import com.yahoo.data.access.simple.Value; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; +import org.junit.jupiter.api.Test; + +import java.util.*; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.*; + +public class GlobalPhaseRerankHitsImplTest { + static class EvalSum implements Evaluator { + double baseValue; + List<Tensor> values = new ArrayList<>(); + EvalSum(double baseValue) { this.baseValue = baseValue; } + @Override public Evaluator bind(String name, Tensor value) { + values.add(value); + return this; + } + @Override public double evaluateScore() { + double result = baseValue; + for (var value: values) { + result += value.asDouble(); + } + return result; + } + } + static FunEvalSpec makeConstSpec(double constValue) { + return new FunEvalSpec(() -> new EvalSum(constValue), Collections.emptyList(), Collections.emptyList()); + } + static FunEvalSpec makeSumSpec(List<String> fromQuery, List<String> fromMF) { + return new FunEvalSpec(() -> new EvalSum(0.0), fromQuery, fromMF); + } + static class ExpectingNormalizer extends Normalizer { + List<Double> expected; + ExpectingNormalizer(List<Double> expected) { + super(100); + this.expected = expected; + } + @Override void normalize() { + double rank = 1; + assertEquals(size, expected.size()); + for (int i = 0; i < size; i++) { + assertEquals(data[i], expected.get(i)); + data[i] = rank; + rank += 1; + } + } + @Override String normalizing() { return "expecting"; } + } + static NormalizerSetup makeNormalizer(String name, List<Double> expected, FunEvalSpec evalSpec) { + return new NormalizerSetup(name, () -> new ExpectingNormalizer(expected), evalSpec); + } + static GlobalPhaseSetup makeFullSetup(FunEvalSpec mainSpec, int rerankCount, + List<String> hiddenMF, List<NormalizerSetup> normalizers) + { + return new GlobalPhaseSetup(mainSpec, rerankCount, hiddenMF, normalizers); + } + static GlobalPhaseSetup makeSimpleSetup(FunEvalSpec mainSpec, int rerankCount) { + return makeFullSetup(mainSpec, rerankCount, Collections.emptyList(), Collections.emptyList()); + } + static GlobalPhaseSetup makeNormSetup(FunEvalSpec mainSpec, List<NormalizerSetup> normalizers) { + return makeFullSetup(mainSpec, 100, Collections.emptyList(), normalizers); + } + static record NamedValue(String name, double value) {} + NamedValue value(String name, double value) { + return new NamedValue(name, value); + } + Query makeQuery(List<NamedValue> inQuery, boolean withPrepare) { + var query = new Query(); + for (var v: inQuery) { + query.getRanking().getFeatures().put(v.name, v.value); + } + if (withPrepare) { + query.getRanking().prepare(); + } + return query; + } + Query makeQuery(List<NamedValue> inQuery) { return makeQuery(inQuery, false); } + Query makeQueryWithPrepare(List<NamedValue> inQuery) { return makeQuery(inQuery, true); } + + static Hit makeHit(String id, double score, FeatureData mf) { + Hit hit = new Hit(id, score); + hit.setField("matchfeatures", mf); + return hit; + } + static Hit hit(String id, double score) { + return makeHit(id, score, FeatureData.empty()); + } + static class HitFactory { + MatchFeatureData mfData; + Map<String,Integer> map = new HashMap<>(); + HitFactory(List<String> mfNames) { + int i = 0; + for (var name: mfNames) { + map.put(name, i++); + } + mfData = new MatchFeatureData(mfNames); + } + Hit create(String id, double score, List<NamedValue> inMF) { + var mf = mfData.addHit(); + for (var v: inMF) { + var idx = map.get(v.name); + assertNotNull(idx); + mf.set(idx, v.value); + } + return makeHit(id, score, new FeatureData(mf)); + } + } + Result makeResult(Query query, List<Hit> hits) { + var result = new Result(query); + result.hits().addAll(hits); + return result; + } + static class Expect { + Map<String,Double> map = new HashMap<>(); + static Expect make(List<Hit> hits) { + var result = new Expect(); + for (var hit : hits) { + result.map.put(hit.getId().stringValue(), hit.getRelevance().getScore()); + } + return result; + } + void verifyScores(Result actual) { + double prev = Double.MAX_VALUE; + assertEquals(actual.hits().size(), map.size()); + for (var hit : actual.hits()) { + var name = hit.getId().stringValue(); + var score = map.get(name); + assertNotNull(score, name); + assertEquals(score.doubleValue(), hit.getRelevance().getScore(), name); + assertTrue(score <= prev); + prev = score; + } + } + } + void verifyHasMF(Result result, String name) { + for (var hit: result.hits()) { + if (hit.getField("matchfeatures") instanceof FeatureData mf) { + assertNotNull(mf.getTensor(name)); + } else { + fail("matchfeatures are missing"); + } + } + } + void verifyDoesNotHaveMF(Result result, String name) { + for (var hit: result.hits()) { + if (hit.getField("matchfeatures") instanceof FeatureData mf) { + assertNull(mf.getTensor(name)); + } else { + fail("matchfeatures are missing"); + } + } + } + void verifyDoesNotHaveMatchFeaturesField(Result result) { + for (var hit: result.hits()) { + assertNull(hit.getField("matchfeatures")); + } + } + @Test void partialRerankWithRescaling() { + var setup = makeSimpleSetup(makeConstSpec(3.0), 2); + var query = makeQuery(Collections.emptyList()); + var result = makeResult(query, List.of(hit("a", 3), hit("b", 4), hit("c", 5), hit("d", 6))); + var expect = Expect.make(List.of(hit("a", 1), hit("b", 2), hit("c", 3), hit("d", 3))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + } + @Test void matchFeaturesCanBePartiallyHidden() { + var setup = makeFullSetup(makeSumSpec(Collections.emptyList(), List.of("public_value", "private_value")), 2, + List.of("private_value"), Collections.emptyList()); + var query = makeQuery(Collections.emptyList()); + var factory = new HitFactory(List.of("public_value", "private_value")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("public_value", 2), value("private_value", 3))), + factory.create("b", 2, List.of(value("public_value", 5), value("private_value", 7))))); + var expect = Expect.make(List.of(hit("a", 5), hit("b", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyHasMF(result, "public_value"); + verifyDoesNotHaveMF(result, "private_value"); + } + @Test void matchFeaturesCanBeRemoved() { + var setup = makeFullSetup(makeSumSpec(Collections.emptyList(), List.of("private_value")), 2, + List.of("private_value"), Collections.emptyList()); + var query = makeQuery(Collections.emptyList()); + var factory = new HitFactory(List.of("private_value")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("private_value", 3))), + factory.create("b", 2, List.of(value("private_value", 7))))); + var expect = Expect.make(List.of(hit("a", 3), hit("b", 7))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyDoesNotHaveMatchFeaturesField(result); + } + @Test void queryFeaturesCanBeUsed() { + var setup = makeSimpleSetup(makeSumSpec(List.of("foo"), List.of("bar")), 2); + var query = makeQuery(List.of(value("query(foo)", 7))); + var factory = new HitFactory(List.of("bar")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("bar", 2))), + factory.create("b", 2, List.of(value("bar", 5))))); + var expect = Expect.make(List.of(hit("a", 9), hit("b", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyHasMF(result, "bar"); + } + @Test void queryFeaturesCanBeUsedWhenPrepared() { + var setup = makeSimpleSetup(makeSumSpec(List.of("foo"), List.of("bar")), 2); + var query = makeQueryWithPrepare(List.of(value("query(foo)", 7))); + var factory = new HitFactory(List.of("bar")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("bar", 2))), + factory.create("b", 2, List.of(value("bar", 5))))); + var expect = Expect.make(List.of(hit("a", 9), hit("b", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyHasMF(result, "bar"); + } + @Test void withNormalizer() { + var setup = makeNormSetup(makeSumSpec(Collections.emptyList(), List.of("bar")), + List.of(makeNormalizer("foo", List.of(115.0, 65.0, 55.0, 45.0, 15.0), makeSumSpec(List.of("x"), List.of("bar"))))); + var query = makeQuery(List.of(value("query(x)", 5))); + var factory = new HitFactory(List.of("bar")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("bar", 10))), + factory.create("b", 2, List.of(value("bar", 40))), + factory.create("c", 3, List.of(value("bar", 50))), + factory.create("d", 4, List.of(value("bar", 60))), + factory.create("e", 5, List.of(value("bar", 110))))); + var expect = Expect.make(List.of(hit("a", 15), hit("b", 44), hit("c", 53), hit("d", 62), hit("e", 111))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java index ee0df3adbfb..b451df87727 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java @@ -13,7 +13,6 @@ import java.util.List; import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.BASIC; import static java.math.BigDecimal.ZERO; -import static java.math.BigDecimal.valueOf; public class MockPricingController implements PricingController { @@ -23,34 +22,35 @@ public class MockPricingController implements PricingController { @Override public Prices priceForApplications(List<ApplicationResources> applicationResources, PricingInfo pricingInfo, Plan plan) { - ApplicationResources resources = applicationResources.get(0); - - BigDecimal listPrice = resources.vcpu().multiply(cpuCost) - .add(resources.memoryGb().multiply(memoryCost) - .add(resources.diskGb().multiply(diskCost)) - .add(resources.enclaveVcpu().multiply(cpuCost) - .add(resources.enclaveMemoryGb().multiply(memoryCost)) - .add(resources.enclaveDiskGb().multiply(diskCost)))); + List<PriceInformation> appPrices = applicationResources.stream() + .map(resources -> { + BigDecimal listPrice = resources.vcpu().multiply(cpuCost) + .add(resources.memoryGb().multiply(memoryCost)) + .add(resources.diskGb().multiply(diskCost)) + .add(resources.enclaveVcpu().multiply(cpuCost)) + .add(resources.enclaveMemoryGb().multiply(memoryCost)) + .add(resources.enclaveDiskGb().multiply(diskCost)); - BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-1.00") : new BigDecimal("8.00"); - BigDecimal listPriceWithSupport = listPrice.add(supportLevelCost); - BigDecimal enclaveDiscount = isEnclave(resources) ? new BigDecimal("-0.15") : BigDecimal.ZERO; - BigDecimal volumeDiscount = new BigDecimal("-0.1"); - BigDecimal appTotalAmount = listPrice.add(supportLevelCost).add(enclaveDiscount).add(volumeDiscount); + BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-1.00") : new BigDecimal("8.00"); + BigDecimal listPriceWithSupport = listPrice.add(supportLevelCost); + BigDecimal enclaveDiscount = isEnclave(resources) ? new BigDecimal("-0.15") : BigDecimal.ZERO; + BigDecimal volumeDiscount = new BigDecimal("-0.1"); + BigDecimal appTotalAmount = listPrice.add(supportLevelCost).add(enclaveDiscount).add(volumeDiscount); - List<PriceInformation> appPrices = applicationResources.stream() - .map(appResources -> new PriceInformation(listPriceWithSupport, - volumeDiscount, - ZERO, - enclaveDiscount, - appTotalAmount)) + return new PriceInformation(listPriceWithSupport, + volumeDiscount, + ZERO, + enclaveDiscount, + appTotalAmount); + }) .toList(); PriceInformation sum = PriceInformation.sum(appPrices); - var committedAmountDiscount = new BigDecimal("-0.2"); + System.out.println(pricingInfo.committedHourlyAmount()); + var committedAmountDiscount = pricingInfo.committedHourlyAmount().compareTo(ZERO) > 0 ? new BigDecimal("-0.2") : ZERO; var totalAmount = sum.totalAmount().add(committedAmountDiscount); var enclave = ZERO; - if (resources.enclave() && totalAmount.compareTo(new BigDecimal("14.00")) < 0) + if (applicationResources.stream().anyMatch(ApplicationResources::enclave) && totalAmount.compareTo(new BigDecimal("14.00")) < 0) enclave = new BigDecimal("14.00").subtract(totalAmount); var totalPrice = new PriceInformation(ZERO, ZERO, committedAmountDiscount, enclave, totalAmount); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 718320c02ca..83cd5dab2f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.messagebus.Message; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -98,6 +99,9 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .post(Slime.class, self::newAdditionalItem)) .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/item/{item}") .delete(self::deleteAdditionalItem)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/plan") + .get(self::accountantTenantPlan) + .post(Slime.class, self::setAccountantTenantPlan)) .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") .put(Slime.class, self::putAccountantInvoiceExport)) .addRoute(RestApi.route("/billing/v2/accountant/plans") @@ -361,6 +365,39 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return slime; } + private MessageResponse setAccountantTenantPlan(RestApi.RequestContext requestContext, Slime body) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var planId = PlanId.from(getInspectorFieldOrThrow(body.get(), "id")); + var response = billing.setPlan(tenant.name(), planId, false, true); + + if (response.isSuccess()) { + return new MessageResponse("Plan: " + planId.value()); + } else { + throw new RestApiException.BadRequest("Could not change plan: " + response.getErrorMessage()); + } + } + + private Slime accountantTenantPlan(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var planId = billing.getPlan(tenant.name()); + var plan = planRegistry.plan(planId); + + if (plan.isEmpty()) { + throw new RestApiException.BadRequest("Plan with ID '" + planId.value() + "' does not exist"); + } + + var slime = new Slime(); + var root = slime.setObject(); + root.setString("id", plan.get().id().value()); + root.setString("name", plan.get().displayName()); + + return slime; + } + // --------- INVOICE RENDERING ---------- private void invoicesSummaryToSlime(Cursor slime, List<Bill> bills) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java index 0a43ec599d5..9a2a57359d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.pricing; import com.yahoo.collections.Pair; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.NodeResources; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; @@ -35,8 +34,6 @@ import java.util.logging.Logger; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.restapi.ErrorResponse.methodNotAllowed; import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel; -import static java.lang.Double.parseDouble; -import static java.lang.Integer.parseInt; import static java.math.BigDecimal.ZERO; import static java.nio.charset.StandardCharsets.UTF_8; @@ -116,41 +113,13 @@ public class PricingApiHandler extends ThreadedHttpRequestHandler { default -> throw new IllegalArgumentException("Unknown query parameter '" + entry.getFirst() + '\''); } } - if (appResources.isEmpty()) throw new IllegalArgumentException("No application resources found in query"); PricingInfo pricingInfo = new PricingInfo(supportLevel, committedSpend); return new PriceParameters(List.of(), pricingInfo, plan, appResources); } - private ClusterResources clusterResources(String resourcesString) { - List<String> elements = Arrays.stream(resourcesString.split(",")).toList(); - - var nodes = 0; - var vcpu = 0d; - var memoryGb = 0d; - var diskGb = 0d; - var gpuMemoryGb = 0d; - - for (var element : keysAndValues(elements)) { - var value = element.getSecond(); - switch (element.getFirst().toLowerCase()) { - case "nodes" -> nodes = parseInt(value); - case "vcpu" -> vcpu = parseDouble(value); - case "memorygb" -> memoryGb = parseDouble(value); - case "diskgb" -> diskGb = parseDouble(value); - case "gpumemorygb" -> gpuMemoryGb = parseDouble(value); - default -> throw new IllegalArgumentException("Unknown resource type '" + element.getFirst() + '\''); - } - } - - var nodeResources = new NodeResources(vcpu, memoryGb, diskGb, 0); // 0 bandwidth, not used in price calculation - if (gpuMemoryGb > 0) - nodeResources = nodeResources.with(new NodeResources.GpuResources(1, gpuMemoryGb)); - return new ClusterResources(nodes, 1, nodeResources); - } - private ApplicationResources applicationResources(String appResourcesString) { - List<String> elements = Arrays.stream(appResourcesString.split(",")).toList(); + List<String> elements = List.of(appResourcesString.split(",")); var vcpu = ZERO; var memoryGb = ZERO; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index a2290f1f664..424b8d84472 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -185,4 +185,30 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { {"message":"Successfully deleted line item line-item-id"}"""); } } + + @Test + void require_current_plan() { + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/plan") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"id":"trial","name":"Free Trial - for testing purposes"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/plan", Request.Method.POST) + .roles(Role.hostedAccountant()) + .data(""" + {"id": "paid"}"""); + tester.assertResponse(accountantRequest, """ + {"message":"Plan: paid"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/plan") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"id":"paid","name":"Paid Plan - for testing purposes"}"""); + } + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java index c4b5a771725..f2ce0dfeef2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java @@ -21,6 +21,12 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { @Test void testPricingInfoBasic() { + tester().assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0"), + """ + { "applications": [ ], "priceInfo": [ ], "totalAmount": "0.00" } + """, + 200); + var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation1App(BASIC)); tester().assertJsonResponse(request, """ { @@ -110,10 +116,8 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { ] } ], - "priceInfo": [ - {"description": "Committed spend", "amount": "-0.20"} - ], - "totalAmount": "25.90" + "priceInfo": [ ], + "totalAmount": "26.10" } """, 200); @@ -128,9 +132,6 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { tester.assertJsonResponse(request("/pricing/v1/pricing?"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: ''\"}", 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application resources found in query\"}", - 400); tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&resources"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: 'resources'\"}", 400); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2b3fe84ec84..c2f6f782dbc 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -340,6 +340,13 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag EXCLUSIVE_PROVISIONING = defineFeatureFlag( + "exclusive-provisioning", false, + List.of("hakonhall"), "2023-10-12", "2023-12-12", + "Whether to provision a host exclusively to an application ID only based on exclusive=\"true\" from services.xml. " + + "Enabling this will produce hosts with exclusiveTo[ApplicationId] without provisionedToApplicationId.", + "Takes immediate effect when provisioning new hosts"); + public static final UnboundBooleanFlag WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "write-config-server-session-data-as-blob", false, List.of("hmusum"), "2023-07-19", "2023-11-01", diff --git a/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java b/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java index b046d55c089..a15f2916091 100644 --- a/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java +++ b/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java @@ -45,8 +45,14 @@ public class MetricSetDocumentation { referenceBuilder.append(String.format(""" --- # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + # Note: This file is generated by + # https://github.com/vespa-engine/vespa/blob/master/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java title: "%s Metric Set" - ---""", name)); + --- + <p> + This document provides reference documentation for the %s metric set, including suffixes present per metric. + If the suffix column contains "N/A" then the base name of the corresponding metric is used with no suffix. + </p>""", name, name)); metricsByType.keySet() .stream() .sorted() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 09d6f96d88e..a876999e80b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -22,14 +22,21 @@ public interface HostProvisioner { enum HostSharing { - /** The host must be provisioned exclusively for the applicationId */ + /** The host must be provisioned exclusively for the application ID. */ + provision, + + /** The host must be exclusive to a single application ID */ exclusive, /** The host must be provisioned to be shared with other applications. */ shared, /** The client has no requirements on whether the host must be provisioned exclusively or shared. */ - any + any; + + public boolean isExclusiveAllocation() { + return this == provision || this == exclusive; + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 0ffd42aedba..89ff0938d59 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -208,7 +208,9 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : HostSharing.any; + return cluster.isExclusive() ? HostSharing.provision : + nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + HostSharing.any; else return HostSharing.any; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 7da80440667..8a84cfef09a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -31,6 +31,7 @@ public class ProvisionedHost { private final Flavor hostFlavor; private final NodeType hostType; private final Optional<ApplicationId> provisionedForApplicationId; + private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final List<HostName> nodeHostnames; private final NodeResources nodeResources; @@ -38,7 +39,9 @@ public class ProvisionedHost { private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, - Optional<ApplicationId> provisionedForApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<ApplicationId> provisionedForApplicationId, + Optional<ApplicationId> exclusiveToApplicationId, + Optional<ClusterSpec.Type> exclusiveToClusterType, List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); @@ -47,6 +50,7 @@ public class ProvisionedHost { this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId must be set"); + this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); this.nodeHostnames = validateNodeAddresses(nodeHostnames); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); @@ -68,6 +72,7 @@ public class ProvisionedHost { .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); provisionedForApplicationId.ifPresent(builder::provisionedForApplicationId); + exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); if ( ! hostTTL.isZero()) builder.hostTTL(hostTTL); return builder.build(); @@ -85,6 +90,7 @@ public class ProvisionedHost { public Flavor hostFlavor() { return hostFlavor; } public NodeType hostType() { return hostType; } public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } + public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } public List<HostName> nodeHostnames() { return nodeHostnames; } public NodeResources nodeResources() { return nodeResources; } @@ -103,6 +109,7 @@ public class ProvisionedHost { hostFlavor.equals(that.hostFlavor) && hostType == that.hostType && provisionedForApplicationId.equals(that.provisionedForApplicationId) && + exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeHostnames.equals(that.nodeHostnames) && nodeResources.equals(that.nodeResources) && @@ -112,7 +119,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); } @Override @@ -123,8 +130,9 @@ public class ProvisionedHost { ", hostFlavor=" + hostFlavor + ", hostType=" + hostType + ", provisionedForApplicationId=" + provisionedForApplicationId + + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeHostnames=" + nodeHostnames + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index def3e003ab3..f7710ca7019 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -78,8 +78,8 @@ public class MockHostProvisioner implements HostProvisioner { Flavor hostFlavor = hostFlavors.get(request.clusterType().orElse(ClusterSpec.Type.content)); if (hostFlavor == null) hostFlavor = flavors.stream() - .filter(f -> request.sharing() == HostSharing.exclusive ? compatible(f, request.resources()) - : satisfies(f, request.resources())) + .filter(f -> request.sharing().isExclusiveAllocation() ? compatible(f, request.resources()) + : satisfies(f, request.resources())) .filter(f -> realHostResourcesWithinLimits.test(f.resources())) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + request.resources(), true)); @@ -91,7 +91,8 @@ public class MockHostProvisioner implements HostProvisioner { hostHostname, hostFlavor, request.type(), - request.sharing() == HostSharing.exclusive ? Optional.of(request.owner()) : Optional.empty(), + request.sharing() == HostSharing.provision ? Optional.of(request.owner()) : Optional.empty(), + request.sharing().isExclusiveAllocation() ? Optional.of(request.owner()) : Optional.empty(), Optional.empty(), createHostnames(request.type(), hostFlavor, index), request.resources(), diff --git a/screwdriver.yaml b/screwdriver.yaml index 6efb9145b09..a3eedc02999 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -34,7 +34,7 @@ shared: du -sh /tmp/vespa/* if [[ -z "$SD_PULL_REQUEST" ]]; then - if [[ -z $VESPA_USE_SANITIZER ]] || [[ $VESPA_USE_SANITIZER == null ]]; then + if [[ -z "$VESPA_USE_SANITIZER" ]] || [[ "$VESPA_USE_SANITIZER" == null ]]; then # Remove what we have produced rm -rf $LOCAL_MVN_REPO/com/yahoo rm -rf $LOCAL_MVN_REPO/ai/vespa diff --git a/vespalib/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index b6094a050d9..0f32aa953a8 100644 --- a/vespalib/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp @@ -202,7 +202,7 @@ FastOS_Linux_File::Write2(const void *buffer, size_t length) if (writtenNow > 0) { written += writtenNow; } else { - return (written > 0) ? written : writtenNow;; + return (written > 0) ? written : writtenNow; } } return written; @@ -239,8 +239,8 @@ FastOS_Linux_File::internalWrite2(const void *buffer, size_t length) } if (writeRes > 0) { _filePointer += writeRes; - if (_filePointer > _cachedSize) { - _cachedSize = _filePointer; + if (_filePointer > _cachedSize.load(std::memory_order_relaxed)) { + _cachedSize.store(_filePointer, std::memory_order_relaxed); } } } else { @@ -277,7 +277,7 @@ FastOS_Linux_File::SetSize(int64_t newSize) bool rc = FastOS_UNIX_File::SetSize(newSize); if (rc) { - _cachedSize = newSize; + _cachedSize.store(newSize, std::memory_order_relaxed); } return rc; } @@ -334,19 +334,21 @@ FastOS_Linux_File::DirectIOPadding (int64_t offset, size_t length, size_t &padBe if (padAfter == _directIOFileAlign) { padAfter = 0; } - if (int64_t(offset+length+padAfter) > _cachedSize) { + int64_t fileSize = _cachedSize.load(std::memory_order_relaxed); + if (int64_t(offset+length+padAfter) > fileSize) { // _cachedSize is not really trustworthy, so if we suspect it is not correct, we correct it. // The main reason is that it will not reflect the file being extended by another filedescriptor. - _cachedSize = getSize(); + fileSize = getSize(); + _cachedSize.store(fileSize, std::memory_order_relaxed); } if ((padAfter != 0) && - (static_cast<int64_t>(offset + length + padAfter) > _cachedSize) && - (static_cast<int64_t>(offset + length) <= _cachedSize)) + (static_cast<int64_t>(offset + length + padAfter) > fileSize) && + (static_cast<int64_t>(offset + length) <= fileSize)) { - padAfter = _cachedSize - (offset + length); + padAfter = fileSize - (offset + length); } - if (static_cast<uint64_t>(offset + length + padAfter) <= static_cast<uint64_t>(_cachedSize)) { + if (static_cast<uint64_t>(offset + length + padAfter) <= static_cast<uint64_t>(fileSize)) { return true; } } diff --git a/vespalib/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h index 1295ce38316..af6e6af51af 100644 --- a/vespalib/src/vespa/fastos/linux_file.h +++ b/vespalib/src/vespa/fastos/linux_file.h @@ -10,21 +10,23 @@ #pragma once #include "unix_file.h" +#include <atomic> /** * This is the Linux implementation of @ref FastOS_File. Most * methods are inherited from @ref FastOS_UNIX_File. */ -class FastOS_Linux_File : public FastOS_UNIX_File +class FastOS_Linux_File final : public FastOS_UNIX_File { public: using FastOS_UNIX_File::ReadBuf; protected: - int64_t _cachedSize; + std::atomic<int64_t> _cachedSize; int64_t _filePointer; // Only maintained/used in directio mode public: - FastOS_Linux_File (const char *filename = nullptr); + FastOS_Linux_File() : FastOS_Linux_File(nullptr) {} + explicit FastOS_Linux_File(const char *filename); ~FastOS_Linux_File () override; bool GetDirectIORestrictions(size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) override; bool DirectIOPadding(int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) override; |