diff options
author | HÃ¥kon Hallingstad <hakon@verizonmedia.com> | 2020-04-20 10:36:33 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-20 10:36:33 +0200 |
commit | b105eead1fbbcefbb85bc962749f2a12fa660bbe (patch) | |
tree | 7a3c996c00b854066d32608a002335715fb98c96 /config-model-api/src | |
parent | f61f6c701dc91e839b865f158a6da56ff166def7 (diff) | |
parent | 9ab0ef70e9ed4f422df67603f26bcb0c7918fdc4 (diff) |
Merge branch 'master' into hakonhall/remove-use-bucket-space-metric-feature-flag
Diffstat (limited to 'config-model-api/src')
11 files changed, 119 insertions, 85 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java index c4dee70cd86..480d12b6700 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java @@ -2,14 +2,16 @@ package com.yahoo.config.application.api; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.*; - +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; import com.yahoo.text.Utf8; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * Metadata about an application package. @@ -27,32 +29,6 @@ public class ApplicationMetaData { private final long generation; private final long previousActiveGeneration; - // TODO: Remove after September 2019 - public ApplicationMetaData(File appDir, - String deployedByUser, - String deployedFromDir, - Long deployTimestamp, - boolean internalRedeploy, - String checksum, - Long generation, - long previousActiveGeneration) { - this(deployedByUser, deployedFromDir, deployTimestamp, internalRedeploy, - appDir.getName(), checksum, generation, previousActiveGeneration); - } - - // TODO: Remove after September 2019 - public ApplicationMetaData(String deployedByUser, String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, - String applicationName, String checksum, Long generation, long previousActiveGeneration) { - this(deployedByUser, - deployedFromDir, - deployTimestamp, - internalRedeploy, - ApplicationId.from(TenantName.defaultName(), ApplicationName.from(applicationName), InstanceName.from("default")), - checksum, - generation, - previousActiveGeneration); - } - public ApplicationMetaData(String deployedByUser, String deployedFromDir, Long deployTimestamp, boolean internalRedeploy, ApplicationId applicationId, String checksum, Long generation, long previousActiveGeneration) { this.deployedByUser = deployedByUser; @@ -66,15 +42,6 @@ public class ApplicationMetaData { } /** - * Gets the name of the application (name of the directory from which application was deployed. - * Will return null if a problem occurred while getting metadata - * - * @return application name - */ - // TODO: Remove after September 2019 - public String getApplicationName() { return applicationId.application().toString(); } - - /** * Gets the user who deployed the application. * Will return null if a problem occurred while getting metadata * @@ -117,10 +84,6 @@ public class ApplicationMetaData { public boolean isInternalRedeploy() { return internalRedeploy; } /** Returns an md5 hash of the contents of the application package */ - // TODO: Remove after September 2019 - public String getCheckSum() { return checksum; } - - /** Returns an md5 hash of the contents of the application package */ public String getChecksum() { return checksum; } /** Returns the previously active generation at the point when this application was created. */ @@ -140,18 +103,11 @@ public class ApplicationMetaData { Inspector deploy = root.field("deploy"); Inspector app = root.field("application"); - // TODO: Simplify to just ApplicationId.fromSerializedForm(app.field("id").asString()) after September 2019 - ApplicationId applicationId = app.field("id").valid() ? - ApplicationId.fromSerializedForm(app.field("id").asString()) : - ApplicationId.from(TenantName.defaultName(), - ApplicationName.from(app.field("name").asString()), - InstanceName.from("default")); - return new ApplicationMetaData(deploy.field("user").asString(), deploy.field("from").asString(), deploy.field("timestamp").asLong(), booleanField("internalRedeploy", false, deploy), - applicationId, + ApplicationId.fromSerializedForm(app.field("id").asString()), app.field("checksum").asString(), app.field("generation").asLong(), app.field("previousActiveGeneration").asLong()); @@ -170,7 +126,6 @@ public class ApplicationMetaData { deploy.setBool("internalRedeploy", internalRedeploy); Cursor app = meta.setObject("application"); app.setString("id", applicationId.serializedForm()); - app.setString("name", applicationId.application().value()); // TODO: Remove after September 2019 app.setString("checksum", checksum); app.setLong("generation", generation); app.setLong("previousActiveGeneration", previousActiveGeneration); @@ -188,7 +143,7 @@ public class ApplicationMetaData { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { new JsonFormat(false).encode(baos, slime); - return baos.toString("UTF-8"); + return baos.toString(StandardCharsets.UTF_8); } catch (IOException e) { throw new RuntimeException("Unable to encode metadata", e); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index db3d391d19b..174f9bb54d7 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -36,8 +36,6 @@ import java.util.jar.JarFile; * The class hides detail as to whether the source is local files or ZooKeeper * data in config server. * - * Anyone wanting to access application data should use this interface. - * * @author Vegard Havdal */ public interface ApplicationPackage { @@ -47,7 +45,8 @@ public interface ApplicationPackage { String HOSTS = "hosts.xml"; String SERVICES = "services.xml"; - Path SEARCH_DEFINITIONS_DIR = Path.fromString("searchdefinitions"); + Path SCHEMAS_DIR = Path.fromString("schemas"); + Path SEARCH_DEFINITIONS_DIR = Path.fromString("searchdefinitions"); // Legacy addition to schemas String COMPONENT_DIR = "components"; String SEARCHCHAINS_DIR = "search/chains"; String DOCPROCCHAINS_DIR = "docproc/chains"; @@ -168,7 +167,7 @@ public interface ApplicationPackage { } /** - * Returns inforamtion about a file + * Returns information about a file * * @param relativePath the relative path of the file within this application package. * @return information abut the file, returned whether or not the file exists @@ -206,23 +205,26 @@ public interface ApplicationPackage { Reader getRankingExpression(String name); /** - * Returns the name-payload pairs of any sd files under path/searchdefinitions/ in the given jar bundle - * @param bundle The jar file, which will be closed afterwards by this method. - * @param path For example 'complex/' + * Returns the name-payload pairs of any sd files under path/schemas and path/searchdefinitions/ + * in the given jar bundle. + * + * @param bundle the jar file, which will be closed afterwards by this method + * @param path for example 'complex/' * @return map of the SD payloads * @throws IOException if it is reading sd files fails */ static Map<String, String> getBundleSdFiles(String path, JarFile bundle) throws IOException { - Map<String,String> ret = new LinkedHashMap<>(); + Map<String, String> schemas = new LinkedHashMap<>(); for (Enumeration<JarEntry> e = bundle.entries(); e.hasMoreElements();) { - JarEntry je=e.nextElement(); - if (je.getName().startsWith(path+SEARCH_DEFINITIONS_DIR+"/") && je.getName().endsWith(SD_NAME_SUFFIX)) { - String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(je))); - ret.put(getFileName(je), contents); + JarEntry entry = e.nextElement(); + if ((entry.getName().startsWith(path + SCHEMAS_DIR + "/") || entry.getName().startsWith(path + SEARCH_DEFINITIONS_DIR + "/")) + && entry.getName().endsWith(SD_NAME_SUFFIX)) { + String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(entry))); + schemas.put(getFileName(entry), contents); } } bundle.close(); - return ret; + return schemas; } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java index e63e4ce3af6..96e76e46cda 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java @@ -10,8 +10,6 @@ public interface ConfigServerSpec { String getHostName(); int getConfigServerPort(); - // TODO: Remove when latest model version in use is 7.47 - default int getHttpPort() { return getConfigServerPort() + 1; } int getZooKeeperPort(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java index bf58000dd36..4edf3c455d0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java @@ -18,15 +18,17 @@ public interface HostProvisioner { // TODO: Remove HostSpec allocateHost(String alias); + @Deprecated // TODO: Remove after April 2020 + List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); + /** * Prepares allocation of a set of hosts with a given type, common id and the amount. * * @param cluster the cluster to allocate nodes to * @param capacity the capacity describing the capacity requested - * @param groups the number of groups to divide the nodes into * @param logger a logger to which messages to the deployer may be written * @return the specification of the allocated hosts */ - List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); + List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 843bce6de7c..8ebb449ceda 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; @@ -26,12 +27,16 @@ public interface ModelContext { Optional<Model> previousModel(); Optional<ApplicationPackage> permanentApplicationPackage(); Optional<HostProvisioner> hostProvisioner(); + Provisioned provisioned(); DeployLogger deployLogger(); ConfigDefinitionRepo configDefinitionRepo(); FileRegistry getFileRegistry(); Properties properties(); default Optional<File> appDir() { return Optional.empty();} + /** The Docker image repo we want to use for images for this deployment (optional, will use default if empty) */ + default Optional<String> wantedDockerImageRepository() { return Optional.empty(); } + /** The Vespa version this model is built for */ Version modelVespaVersion(); @@ -51,15 +56,40 @@ public interface ModelContext { Set<ContainerEndpoint> endpoints(); boolean isBootstrap(); boolean isFirstTimeDeployment(); - // TODO: Remove when Vespa 7.112 is the oldest config model in use + + // TODO: Only needed for LbServicesProducerTest default boolean useDedicatedNodeForLogserver() { return true; } + + // TODO Revisit in May or June 2020 boolean useAdaptiveDispatch(); - // TODO: Remove temporary default implementations + + // TODO: Remove after April 2020 default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); } + default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); } + + // TODO Revisit in May or June 2020 double defaultTermwiseLimit(); + + // TODO Revisit in May or June 2020 + double defaultSoftStartSeconds(); + + // TODO Revisit in May or June 2020 + double defaultTopKProbability(); + // TODO: Remove once there are no Vespa versions below 7.170 boolean useBucketSpaceMetric(); + + default boolean useNewAthenzFilter() { return true; } // TODO bjorncs: Remove after end of April + + // TODO: Remove after April 2020 + default boolean usePhraseSegmenting() { return false; } + + default String proxyProtocol() { return "https-only"; } + default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); } + + // TODO(mpolden): Remove after May 2020 + default boolean useDedicatedNodesWhenUnspecified() { return true; } } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java new file mode 100644 index 00000000000..e4201836e57 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java @@ -0,0 +1,28 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A recording of the capacity requests issued during a model build. + * Requests are only recorded here if provision requests are issued to the node repo. + * + * @author bratseth + */ +public class Provisioned { + + private final Map<ClusterSpec.Id, Capacity> provisioned = new HashMap<>(); + + public void add(ClusterSpec.Id id, Capacity capacity) { + provisioned.put(id, capacity); + } + + /** Returns an unmodifiable map of all the provision requests recorded during build of the model this belongs to */ + public Map<ClusterSpec.Id, Capacity> all() { return Collections.unmodifiableMap(provisioned); } + +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java index 1735e08c930..50f1ea2336d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java @@ -20,13 +20,15 @@ import java.util.Set; public class SuperModel { private final Map<ApplicationId, ApplicationInfo> models; + private final boolean complete; public SuperModel() { - this.models = Collections.emptyMap(); + this(Collections.emptyMap(), false); } - public SuperModel(Map<ApplicationId, ApplicationInfo> models) { + public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) { this.models = models; + this.complete = complete; } public Map<TenantName, Set<ApplicationInfo>> getModelsPerTenant() { @@ -45,6 +47,8 @@ public class SuperModel { return ImmutableMap.copyOf(models); } + public boolean isComplete() { return complete; } + public List<ApplicationInfo> getAllApplicationInfos() { return new ArrayList<>(models.values()); } @@ -57,17 +61,19 @@ public class SuperModel { public SuperModel cloneAndSetApplication(ApplicationInfo application) { Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models); newModels.put(application.getApplicationId(), application); - - return new SuperModel(newModels); + return new SuperModel(newModels, complete); } public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) { Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models); newModels.remove(applicationId); - - return new SuperModel(newModels); + return new SuperModel(newModels, complete); } + public SuperModel cloneAsComplete() { return new SuperModel(models, true); } + + public Set<ApplicationId> getApplicationIds() { return models.keySet(); } + private static Map<ApplicationId, ApplicationInfo> cloneModels(Map<ApplicationId, ApplicationInfo> models) { return new LinkedHashMap<>(models); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java index 497c38af908..930c3c94907 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; * Interface for those wanting to be notified about changes to the SuperModel. */ public interface SuperModelListener { + /** * Application has been activated: Either deployed the first time, * internally redeployed, or externally triggered redeploy. @@ -17,4 +18,12 @@ public interface SuperModelListener { * Application has been removed. */ void applicationRemoved(SuperModel superModel, ApplicationId id); + + /** + * Invoked once all applications that were supposed to be deployed on bootstrap + * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo) + * applicationActivated} have been invoked). The SuperModel is then said to be "complete". + */ + void notifyOfCompleteness(SuperModel superModel); + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java index a3478026520..689e2524dde 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi // Not really "public", only annotated as such to enable the ABI checker plugin package com.yahoo.config.model.api; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 5bf103d1836..32d903786dc 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -1002,14 +1002,14 @@ public class DeploymentSpecTest { public void notificationsDefault() { StringReader r = new StringReader( "<deployment version='1.0'>" + - " <notifications when=\"failing-commit\">" + - " <email role=\"author\"/>" + + " <notifications>" + + " <email role=\"author\" when=\"failing\"/>" + " <email address=\"mary@dev\"/>" + " </notifications>" + " <instance id='instance1'>" + " <notifications when=\"failing\">" + " <email role=\"author\"/>" + - " <email address=\"john@operator\"/>" + + " <email address=\"john@operator\" when=\"failing-commit\"/>" + " </notifications>" + " </instance>" + " <instance id='instance2'>" + @@ -1020,9 +1020,13 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); DeploymentInstanceSpec instance1 = spec.requireInstance("instance1"); assertEquals(Set.of(author), instance1.notifications().emailRolesFor(failing)); - assertEquals(Set.of("john@operator"), instance1.notifications().emailAddressesFor(failing)); + assertEquals(Set.of(), instance1.notifications().emailAddressesFor(failing)); + assertEquals(Set.of(author), instance1.notifications().emailRolesFor(failingCommit)); + assertEquals(Set.of("john@operator"), instance1.notifications().emailAddressesFor(failingCommit)); DeploymentInstanceSpec instance2 = spec.requireInstance("instance2"); + assertEquals(Set.of(author), instance2.notifications().emailRolesFor(failing)); + assertEquals(Set.of(), instance2.notifications().emailAddressesFor(failing)); assertEquals(Set.of(author), instance2.notifications().emailRolesFor(failingCommit)); assertEquals(Set.of("mary@dev"), instance2.notifications().emailAddressesFor(failingCommit)); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index 6a815a467f5..4a7ef7b43f7 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -318,6 +318,7 @@ public class DeploymentSpecWithoutInstanceTest { @Test public void testEmpty() { + assertEquals(DeploymentSpec.empty, DeploymentSpec.fromXml("<deployment version='1.0'>\n</deployment>")); assertEquals(0, DeploymentSpec.empty.steps().size()); assertTrue(DeploymentSpec.empty.athenzDomain().isEmpty()); assertTrue(DeploymentSpec.empty.athenzService().isEmpty()); |