diff options
15 files changed, 123 insertions, 64 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index 36d6efdf59b..d262c7bc862 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -27,21 +27,21 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Check whether or not this file is a directory. + * Checks whether this file is a directory. * * @return true if it is, false if not. */ public abstract boolean isDirectory(); /** - * Test whether or not this file exists. + * Tests whether this file exists. * * @return true if it exists, false if not. */ public abstract boolean exists(); /** - * Create a {@link Reader} for the contents of this file. + * Creates a {@link Reader} for the contents of this file. * * @return A {@link Reader} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -50,7 +50,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { /** - * Create an {@link InputStream} for the contents of this file. + * Creates an {@link InputStream} for the contents of this file. * * @return An {@link InputStream} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -58,7 +58,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract InputStream createInputStream() throws FileNotFoundException; /** - * Create a directory at the path represented by this file. Parent directories will + * Creates a directory at the path represented by this file. Parent directories will * be automatically created. * * @return this @@ -67,7 +67,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile createDirectory(); /** - * Write the contents from this reader to this file. Any existing content will be overwritten! + * Writes the contents from supplied reader to this file. Any existing content will be overwritten! * * @param input A reader pointing to the content that should be written. * @return this @@ -82,7 +82,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile appendFile(String value); /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @return a list of files in this directory. @@ -92,7 +92,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is a file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @param filter A filter functor for filtering path names @@ -101,7 +101,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract List<ApplicationFile> listFiles(PathFilter filter); /** - * List the files in this directory, optionally list files for subdirectories recursively as well. + * Lists the files in this directory, optionally lists files for subdirectories recursively as well. * * @param recurse Set to true if all files in the directory tree should be returned. * @return a list of files in this directory. @@ -121,7 +121,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Delete the file pointed to by this. If it is a non-empty directory, the operation will throw. + * Deletes the file pointed to by this. If this is a non-empty directory, the operation will throw. * * @return this. * @throws RuntimeException if the file is a directory and not empty. @@ -129,7 +129,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile delete(); /** - * Get the path that this file represents. + * Gets the path that this file represents. * * @return a Path */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9821f3b9568..f434d056bfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -166,7 +166,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger(), deployState.featureFlags(), - userConfiguredUrls); + userConfiguredUrls, + deployState.getApplicationPackage()); for (Component<?, ?> component : getAllComponents()) { files.register(component); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index d2faff7850b..b14495756c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -9,6 +9,7 @@ import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.component.SimpleComponent; import java.util.ArrayList; @@ -24,13 +25,11 @@ import java.util.TreeSet; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster<?> cluster; - private volatile boolean isHostedVespa; private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); private final SortedSet<String> ignoredUserAgentsList = new TreeSet<>(); public JettyHttpServer(String componentId, ContainerCluster<?> cluster, DeployState deployState) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); - this.isHostedVespa = deployState.isHosted(); this.cluster = cluster; FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId); addChild(filterBindingsProviderComponent); @@ -42,8 +41,6 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro } } - public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } - public void addConnector(ConnectorFactory connectorFactory) { connectorFactories.add(connectorFactory); addChild(connectorFactory); @@ -64,10 +61,8 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .ignoredUserAgents(ignoredUserAgentsList) .searchHandlerPaths(List.of("/search")) ); - if (isHostedVespa) { - // Enable connection log hosted Vespa + if (cluster.getAllComponents().stream().anyMatch(c -> c instanceof ConnectionLogComponent)) builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); - } configureJettyThreadpool(builder); builder.stopTimeout(300); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7653d814d8a..119a3ad18c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -3,19 +3,14 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; -import static com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType.jsonAccessLog; - /** * Builds the config model for the standalone config server. * @@ -57,12 +52,6 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { } @Override - protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { - super.addHttp(deployState, spec, cluster, context); - cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); - } - - @Override protected void addModelEvaluationRuntime(ApplicationContainerCluster cluster) { // Model evaluation bundles are pre-installed in the standalone container. } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 47ae2f40414..a454c1141ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -24,6 +26,7 @@ import java.util.Optional; import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; +import static java.util.logging.Level.WARNING; /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. @@ -37,14 +40,17 @@ public class UserConfiguredFiles implements Serializable { private final DeployLogger logger; private final UserConfiguredUrls userConfiguredUrls; private final String unknownConfigDefinition; + private final ApplicationPackage applicationPackage; public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, ModelContext.FeatureFlags featureFlags, - UserConfiguredUrls userConfiguredUrls) { + UserConfiguredUrls userConfiguredUrls, + ApplicationPackage applicationPackage) { this.fileRegistry = fileRegistry; this.logger = logger; this.userConfiguredUrls = userConfiguredUrls; this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); + this.applicationPackage = applicationPackage; } /** @@ -69,7 +75,7 @@ public class UserConfiguredFiles implements Serializable { if (configDefinition == null) { String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config"; switch (unknownConfigDefinition) { - case "warning" -> logger.logApplicationPackage(Level.WARNING, message); + case "warning" -> logger.logApplicationPackage(WARNING, message); case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key); } return; @@ -155,9 +161,9 @@ public class UserConfiguredFiles implements Serializable { path = Path.fromString(builder.getValue()); } - File file = path.toFile(); - if (file.isDirectory() && (file.listFiles() == null || file.listFiles().length == 0)) - throw new IllegalArgumentException("Directory '" + path.getRelative() + "' is empty"); + ApplicationFile file = applicationPackage.getFile(path); + if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty())) + logger.logApplicationPackage(WARNING, "Directory '" + path.getRelative() + "' is empty"); FileReference reference = registeredFiles.get(path); if (reference == null) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f2e4ec052cb..a38a29893e0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -16,6 +16,7 @@ import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.container.logging.VespaAccessLog; +import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import org.junit.jupiter.api.Test; @@ -129,6 +130,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertEquals("default", config.cluster()); assertEquals(-1, config.queueSize()); assertEquals(256 * 1024, config.bufferSize()); + assertTrue(root.getConfig(ServerConfig.class, "default/container.0/DefaultHttpServer").connectionLog().enabled()); } @Test @@ -141,6 +143,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); Component<?, ?> fileConnectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); + assertFalse(root.getConfig(ServerConfig.class, "default/container.0/DefaultHttpServer").connectionLog().enabled()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index 523b0e74be1..b4a54548062 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -5,11 +5,15 @@ import com.yahoo.config.FileNode; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.schema.processing.ReservedRankingExpressionFunctionNamesTestCase; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -19,12 +23,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -69,12 +75,20 @@ public class UserConfiguredFilesTest { public String toString() { return export().toString(); } } - private UserConfiguredFiles userConfiguredFiles() { + return userConfiguredFiles(new MockApplicationPackage.Builder().build()); + } + + private UserConfiguredFiles userConfiguredFiles(ApplicationPackage applicationPackage) { + return userConfiguredFiles(applicationPackage, new BaseDeployLogger()); + } + + private UserConfiguredFiles userConfiguredFiles(ApplicationPackage applicationPackage, DeployLogger deployLogger) { return new UserConfiguredFiles(fileRegistry, - new BaseDeployLogger(), + deployLogger, new TestProperties(), - new ApplicationContainerCluster.UserConfiguredUrls()); + new ApplicationContainerCluster.UserConfiguredUrls(), + applicationPackage); } @BeforeEach @@ -289,18 +303,42 @@ public class UserConfiguredFilesTest { } @Test - void require_that_using_empty_dir_gives_sane_error_message(@TempDir Path tempDir) { - String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target")); + void require_that_using_empty_dir_fails(@TempDir Path tempDir) { + String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target") + 7); + ApplicationPackage applicationPackage = + new MockApplicationPackage.Builder() + .withRoot(tempDir.toFile().getParentFile()) + .withFiles(Map.of(com.yahoo.path.Path.fromString(tempDir.toFile().getAbsolutePath()), "")) + .build(); + + var logger = new TestDeployLogger(); + def.addPathDef("pathVal"); + builder.setField("pathVal", relativeTempDir); + fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); + userConfiguredFiles(applicationPackage, logger).register(producer); + assertEquals("Directory '" + relativeTempDir + "' is empty", logger.log); + } + + @Test + void require_that_using_non_existing_dir_fails() { + String relativeTempDir = "non-existing"; try { def.addPathDef("pathVal"); builder.setField("pathVal", relativeTempDir); - fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Invalid config in services.xml for 'mynamespace.myname': Directory '" + relativeTempDir + "' is empty", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': No such file or directory '" + relativeTempDir + "'", e.getMessage()); } } + private static class TestDeployLogger implements DeployLogger { + public String log = ""; + @Override + public void log(Level level, String message) { + log += message; + } + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index ed0f9aac884..d0b4ad9e917 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -2,8 +2,6 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import java.util.Objects; import java.util.Optional; @@ -79,6 +77,7 @@ public final class ClusterSpec { return combinedId; } + /** * Returns whether the physical hosts running the nodes of this application can * also run nodes of other applications. Using exclusive nodes for containers increases security and cost. @@ -96,12 +95,6 @@ public final class ClusterSpec { return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); } - // TODO: Remove after July 2023 - @Deprecated - public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); - } - /** Creates a ClusterSpec when requesting a cluster */ public static Builder request(Type type, Id id) { return new Builder(type, id); @@ -121,6 +114,7 @@ public final class ClusterSpec { private Optional<DockerImage> dockerImageRepo = Optional.empty(); private Version vespaVersion; private boolean exclusive = false; + private boolean provisionForApplication = false; private Optional<Id> combinedId = Optional.empty(); private ZoneEndpoint zoneEndpoint = ZoneEndpoint.defaultEndpoint; private boolean stateful; @@ -155,6 +149,11 @@ public final class ClusterSpec { return this; } + public Builder provisionForApplication(boolean provisionForApplication) { + this.provisionForApplication = provisionForApplication; + return this; + } + public Builder combinedId(Optional<Id> combinedId) { this.combinedId = combinedId; return this; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2680b4babb1..3de9d5aef4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -660,11 +660,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye log.log(Level.FINE, () -> "Remove unused file references last modified before " + instant); List<String> fileReferencesToDelete = sortedUnusedFileReferences(fileDirectory.getRoot(), fileReferencesInUse, instant); - if (fileReferencesToDelete.size() > 0) { - log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete); - fileReferencesToDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); + // Do max 20 at a time + var toDelete = fileReferencesToDelete.subList(0, Math.min(fileReferencesToDelete.size(), 20)); + if (toDelete.size() > 0) { + log.log(Level.FINE, () -> "Will delete file references not in use: " + toDelete); + toDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); + log.log(Level.FINE, () -> "Deleted " + toDelete.size() + " file references not in use"); } - return fileReferencesToDelete; + return toDelete; } private boolean isFileReferenceInUse(FileReference fileReference) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2b3fe84ec84..c2f6f782dbc 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -340,6 +340,13 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag EXCLUSIVE_PROVISIONING = defineFeatureFlag( + "exclusive-provisioning", false, + List.of("hakonhall"), "2023-10-12", "2023-12-12", + "Whether to provision a host exclusively to an application ID only based on exclusive=\"true\" from services.xml. " + + "Enabling this will produce hosts with exclusiveTo[ApplicationId] without provisionedToApplicationId.", + "Takes immediate effect when provisioning new hosts"); + public static final UnboundBooleanFlag WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "write-config-server-session-data-as-blob", false, List.of("hmusum"), "2023-07-19", "2023-11-01", diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 09d6f96d88e..a876999e80b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -22,14 +22,21 @@ public interface HostProvisioner { enum HostSharing { - /** The host must be provisioned exclusively for the applicationId */ + /** The host must be provisioned exclusively for the application ID. */ + provision, + + /** The host must be exclusive to a single application ID */ exclusive, /** The host must be provisioned to be shared with other applications. */ shared, /** The client has no requirements on whether the host must be provisioned exclusively or shared. */ - any + any; + + public boolean isExclusiveAllocation() { + return this == provision || this == exclusive; + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 0ffd42aedba..89ff0938d59 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -208,7 +208,9 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : HostSharing.any; + return cluster.isExclusive() ? HostSharing.provision : + nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + HostSharing.any; else return HostSharing.any; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 7da80440667..8a84cfef09a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -31,6 +31,7 @@ public class ProvisionedHost { private final Flavor hostFlavor; private final NodeType hostType; private final Optional<ApplicationId> provisionedForApplicationId; + private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final List<HostName> nodeHostnames; private final NodeResources nodeResources; @@ -38,7 +39,9 @@ public class ProvisionedHost { private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, - Optional<ApplicationId> provisionedForApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<ApplicationId> provisionedForApplicationId, + Optional<ApplicationId> exclusiveToApplicationId, + Optional<ClusterSpec.Type> exclusiveToClusterType, List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); @@ -47,6 +50,7 @@ public class ProvisionedHost { this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId must be set"); + this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); this.nodeHostnames = validateNodeAddresses(nodeHostnames); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); @@ -68,6 +72,7 @@ public class ProvisionedHost { .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); provisionedForApplicationId.ifPresent(builder::provisionedForApplicationId); + exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); if ( ! hostTTL.isZero()) builder.hostTTL(hostTTL); return builder.build(); @@ -85,6 +90,7 @@ public class ProvisionedHost { public Flavor hostFlavor() { return hostFlavor; } public NodeType hostType() { return hostType; } public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } + public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } public List<HostName> nodeHostnames() { return nodeHostnames; } public NodeResources nodeResources() { return nodeResources; } @@ -103,6 +109,7 @@ public class ProvisionedHost { hostFlavor.equals(that.hostFlavor) && hostType == that.hostType && provisionedForApplicationId.equals(that.provisionedForApplicationId) && + exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeHostnames.equals(that.nodeHostnames) && nodeResources.equals(that.nodeResources) && @@ -112,7 +119,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); } @Override @@ -123,8 +130,9 @@ public class ProvisionedHost { ", hostFlavor=" + hostFlavor + ", hostType=" + hostType + ", provisionedForApplicationId=" + provisionedForApplicationId + + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeHostnames=" + nodeHostnames + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index def3e003ab3..f7710ca7019 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -78,8 +78,8 @@ public class MockHostProvisioner implements HostProvisioner { Flavor hostFlavor = hostFlavors.get(request.clusterType().orElse(ClusterSpec.Type.content)); if (hostFlavor == null) hostFlavor = flavors.stream() - .filter(f -> request.sharing() == HostSharing.exclusive ? compatible(f, request.resources()) - : satisfies(f, request.resources())) + .filter(f -> request.sharing().isExclusiveAllocation() ? compatible(f, request.resources()) + : satisfies(f, request.resources())) .filter(f -> realHostResourcesWithinLimits.test(f.resources())) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + request.resources(), true)); @@ -91,7 +91,8 @@ public class MockHostProvisioner implements HostProvisioner { hostHostname, hostFlavor, request.type(), - request.sharing() == HostSharing.exclusive ? Optional.of(request.owner()) : Optional.empty(), + request.sharing() == HostSharing.provision ? Optional.of(request.owner()) : Optional.empty(), + request.sharing().isExclusiveAllocation() ? Optional.of(request.owner()) : Optional.empty(), Optional.empty(), createHostnames(request.type(), hostFlavor, index), request.resources(), diff --git a/screwdriver.yaml b/screwdriver.yaml index 6efb9145b09..a3eedc02999 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -34,7 +34,7 @@ shared: du -sh /tmp/vespa/* if [[ -z "$SD_PULL_REQUEST" ]]; then - if [[ -z $VESPA_USE_SANITIZER ]] || [[ $VESPA_USE_SANITIZER == null ]]; then + if [[ -z "$VESPA_USE_SANITIZER" ]] || [[ "$VESPA_USE_SANITIZER" == null ]]; then # Remove what we have produced rm -rf $LOCAL_MVN_REPO/com/yahoo rm -rf $LOCAL_MVN_REPO/ai/vespa |