diff options
113 files changed, 1581 insertions, 617 deletions
diff --git a/client/go/cmd/log.go b/client/go/cmd/log.go index 4577e890959..7d3f6e95cd8 100644 --- a/client/go/cmd/log.go +++ b/client/go/cmd/log.go @@ -32,6 +32,8 @@ var logCmd = &cobra.Command{ Long: `Show the Vespa log. The logs shown can be limited to a relative or fixed period. All timestamps are shown in UTC. + +Logs for the past hour are shown if no arguments are given. `, Example: `$ vespa log 1h $ vespa log --nldequote=false 10m @@ -68,11 +70,13 @@ $ vespa log --follow`, } func parsePeriod(args []string) (time.Time, time.Time, error) { - if len(args) == 1 { - if fromArg != "" || toArg != "" { - return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) + relativePeriod := fromArg == "" || toArg == "" + if relativePeriod { + period := "1h" + if len(args) > 0 { + period = args[0] } - d, err := time.ParseDuration(args[0]) + d, err := time.ParseDuration(period) if err != nil { return time.Time{}, time.Time{}, err } @@ -82,6 +86,8 @@ func parsePeriod(args []string) (time.Time, time.Time, error) { to := time.Now() from := to.Add(d) return from, to, nil + } else if len(args) > 0 { + return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) } from, err := time.Parse(time.RFC3339, fromArg) if err != nil { diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index 0d77792de6d..44d729c802e 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -3,8 +3,7 @@ package ai.vespa.reindexing; import ai.vespa.reindexing.Reindexing.Status; import ai.vespa.reindexing.Reindexing.Trigger; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.document.DocumentType; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.document.DocumentTypeManager; import com.yahoo.path.Path; import com.yahoo.slime.Cursor; @@ -18,7 +17,6 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -87,7 +85,7 @@ public class ReindexingCurator { try { return curator.lock(lockPath(cluster), lockTimeout); } - catch (UncheckedTimeoutException e) { // TODO jonmv: Avoid use of guava classes. + catch (UncheckedTimeoutException e) { throw new ReindexingLockException(e); } } diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index 25aaa0bce13..654481aee33 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.utils.staterestapi.server; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import java.util.logging.Level; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index b6b9190fedf..87a84911d3e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -10,8 +10,18 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; import com.yahoo.vespa.model.VespaModel; - +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -19,12 +29,15 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.logging.Level; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * A validator for bundles. Uses BND library for some of the validation. @@ -40,29 +53,31 @@ public class BundleValidator extends Validator { public void validate(VespaModel model, DeployState deployState) { ApplicationPackage app = deployState.getApplicationPackage(); for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { + Path path = Path.fromString(info.getPathRelativeToAppDir()); try { - Path path = Path.fromString(info.getPathRelativeToAppDir()); DeployLogger deployLogger = deployState.getDeployLogger(); deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path)); JarFile jarFile = new JarFile(app.getFileReference(path)); validateJarFile(deployLogger, jarFile); } catch (IOException e) { throw new IllegalArgumentException( - "Failed to validate JAR file '" + info.getPathRelativeToAppDir() + "'", e); + "Failed to validate JAR file '" + path.last() + "'", e); } } } void validateJarFile(DeployLogger deployLogger, JarFile jarFile) throws IOException { Manifest manifest = jarFile.getManifest(); - String jarPath = jarFile.getName(); + String filename = Paths.get(jarFile.getName()).getFileName().toString(); if (manifest == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarPath); + throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename); } - validateManifest(deployLogger, jarPath, manifest); + validateManifest(deployLogger, filename, manifest); + getPomXmlContent(deployLogger, jarFile) + .ifPresent(pomXml -> validatePomXml(deployLogger, filename, pomXml)); } - void validateManifest(DeployLogger deployLogger, String jarPath, Manifest mf) { + private void validateManifest(DeployLogger deployLogger, String filename, Manifest mf) { // Check for required OSGI headers Attributes attributes = mf.getMainAttributes(); HashSet<String> mfAttributes = new HashSet<>(); @@ -74,35 +89,37 @@ public class BundleValidator extends Validator { for (String header : requiredOSGIHeaders) { if (!mfAttributes.contains(header)) { throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + jarPath + "'"); + "' was not found in manifest in '" + filename + "'"); } } if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarPath + + deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + filename + ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); } if (attributes.getValue("Import-Package") != null) { - validateImportedPackages(deployLogger, jarPath, mf); + validateImportedPackages(deployLogger, filename, mf); } } - private static void validateImportedPackages(DeployLogger deployLogger, String jarPath, Manifest manifest) { + private static void validateImportedPackages(DeployLogger deployLogger, String filename, Manifest manifest) { Domain osgiHeaders = Domain.domain(manifest); Parameters importPackage = osgiHeaders.getImportPackage(); - Map<DeprecatedArtifact, List<String>> deprecatedPackagesInUse = new HashMap<>(); + Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); importPackage.forEach((packageName, attrs) -> { VersionRange versionRange = attrs.getVersion() != null ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) : null; - for (DeprecatedArtifact deprecatedArtifact : DeprecatedArtifact.values()) { - if (deprecatedArtifact.javaPackages.contains(packageName) - && (versionRange == null || deprecatedArtifact.versionDiscriminator.test(versionRange))) { - deprecatedPackagesInUse.computeIfAbsent(deprecatedArtifact, __ -> new ArrayList<>()) - .add(packageName); + for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { + for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { + if (matcher.test(packageName) + && (versionRange == null || deprecatedBundle.versionDiscriminator.test(versionRange))) { + deprecatedPackagesInUse.computeIfAbsent(deprecatedBundle, __ -> new ArrayList<>()) + .add(packageName); + } } } }); @@ -112,31 +129,94 @@ public class BundleValidator extends Validator { String.format("For JAR file '%s': \n" + "Manifest imports the following Java packages from '%s': %s. \n" + "%s", - jarPath, artifact.name, packagesInUse, artifact.description)); + filename, artifact.name, packagesInUse, artifact.description)); }); } - private enum DeprecatedArtifact { + private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); + + private Optional<String> getPomXmlContent(DeployLogger deployLogger, JarFile jarFile) { + return jarFile.stream() + .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) + .findFirst() + .map(f -> { + try { + return new String(jarFile.getInputStream(f).readAllBytes()); + } catch (IOException e) { + deployLogger.log(Level.INFO, + String.format("Unable to read '%s' from '%s'", f.getName(), jarFile.getName())); + return null; + } + }); + } + + private void validatePomXml(DeployLogger deployLogger, String jarFilename, String pomXmlContent) { + try { + Document pom = DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + .parse(new InputSource(new StringReader(pomXmlContent))); + NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() + .compile("/project/dependencies/dependency") + .evaluate(pom, XPathConstants.NODESET); + for (int i = 0; i < dependencies.getLength(); i++) { + Element dependency = (Element) dependencies.item(i); + String groupId = dependency.getElementsByTagName("groupId").item(0).getTextContent(); + String artifactId = dependency.getElementsByTagName("artifactId").item(0).getTextContent(); + for (DeprecatedMavenArtifact deprecatedArtifact : DeprecatedMavenArtifact.values()) { + if (groupId.equals(deprecatedArtifact.groupId) && artifactId.equals(deprecatedArtifact.artifactId)) { + deployLogger.logApplicationPackage(Level.WARNING, + String.format( + "The pom.xml of bundle '%s' includes a dependency to the artifact '%s:%s'. \n%s", + jarFilename, groupId, artifactId, deprecatedArtifact.description)); + } + } + } + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (Exception e) { + deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", jarFilename)); + } + } + + private enum DeprecatedMavenArtifact { + VESPA_HTTP_CLIENT_EXTENSION("com.yahoo.vespa", "vespa-http-client-extensions", + "This artifact will be removed in Vespa 8. " + + "Programmatic use can be safely removed from system/staging tests. " + + "See internal Vespa 8 release notes for details."); + + final String groupId; + final String artifactId; + final String description; + + DeprecatedMavenArtifact(String groupId, String artifactId, String description) { + this.groupId = groupId; + this.artifactId = artifactId; + this.description = description; + } + } + + private enum DeprecatedProvidedBundle { ORG_JSON("org.json:json", "The org.json library will no longer provided by jdisc runtime on Vespa 8. " + "See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", - Set.of("org.json")); + Set.of("org\\.json")); final String name; - final Collection<String> javaPackages; + final Collection<Predicate<String>> javaPackageMatchers; final Predicate<VersionRange> versionDiscriminator; final String description; - DeprecatedArtifact(String name, String description, Collection<String> javaPackages) { - this(name, description, __ -> true, javaPackages); + DeprecatedProvidedBundle(String name, String description, Collection<String> javaPackagePatterns) { + this(name, description, __ -> true, javaPackagePatterns); } - DeprecatedArtifact(String name, - String description, - Predicate<VersionRange> versionDiscriminator, - Collection<String> javaPackages) { + DeprecatedProvidedBundle(String name, + String description, + Predicate<VersionRange> versionDiscriminator, + Collection<String> javaPackagePatterns) { this.name = name; - this.javaPackages = javaPackages; + this.javaPackageMatchers = javaPackagePatterns.stream() + .map(s -> Pattern.compile(s).asMatchPredicate()) + .collect(Collectors.toList()); this.versionDiscriminator = versionDiscriminator; this.description = description; } diff --git a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF index 760a9ecf00f..760a9ecf00f 100644 --- a/config-model/src/test/cfg/application/validation/testjars/manifest-producing-import-warnings.MF +++ b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF diff --git a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar b/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar Binary files differdeleted file mode 100644 index 84781c4802e..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar b/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar Binary files differdeleted file mode 100644 index f4f7dd4e127..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok.jar b/config-model/src/test/cfg/application/validation/testjars/ok.jar Binary files differdeleted file mode 100644 index fce043c6ff7..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/ok.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..53773b8b1cc --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: ok +Bundle-SymbolicName: ok +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..1f88a5e6477 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: mybundle +Bundle-SymbolicName: mybundle +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml new file mode 100644 index 00000000000..1d28f307824 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.test</groupId> + <artifactId>mybundle</artifactId> + <packaging>container-plugin</packaging> + <version>1.0.0</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-http-client-extensions</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project>
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar Binary files differdeleted file mode 100644 index a395a52d17d..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..21c58490c13 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Archiver-Version: Plexus Archiver +Created-By: vespa container maven plugin +Built-By: tonyv +Build-Jdk: 1.6.0_26 +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/jrt-5.1-SNAPSHOT.jar +Bundle-Version: 5.1.0.SNAPSHOT +Bundle-Name: container maven plugin test +Bundle-ManifestVersion: 2 +Bundle-SymbolicName: TestBundle + diff --git a/config-model/src/test/cfg/application/validation/testjars/test.jar b/config-model/src/test/cfg/application/validation/testjars/test.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/test.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar Binary files differdeleted file mode 100644 index 31266f1e8f2..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar +++ /dev/null diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 474013c17fc..41c7f6d72e2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -397,7 +397,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); - assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize()); + assertEquals(16, qrStartConfig.jvm().baseMaxDirectMemorySize()); assertReindexingConfigPresent(model); assertReindexingConfiguredOnAdminCluster(model); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index e2eae30d78d..ef4353d02fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -3,36 +3,41 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.jar.Manifest; +import java.util.jar.JarOutputStream; +import static com.yahoo.yolean.Exceptions.uncheck; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class BundleValidatorTest { - private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); @Test public void basicBundleValidation() throws Exception { // Valid jar file - JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar")); + JarFile ok = createTemporaryJarFile("ok"); BundleValidator bundleValidator = new BundleValidator(); bundleValidator.validateJarFile(new BaseDeployLogger(), ok); // No manifest - validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar"); + validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); } private void validateWithException(String jarName, String exceptionMessage) throws IOException { try { - JarFile jarFile = new JarFile(JARS_DIR + jarName); + JarFile jarFile = createTemporaryJarFile(jarName); BundleValidator bundleValidator = new BundleValidator(); bundleValidator.validateJarFile(new BaseDeployLogger(), jarFile); assert (false); @@ -46,8 +51,8 @@ public class BundleValidatorTest { final StringBuffer buffer = new StringBuffer(); DeployLogger logger = createDeployLogger(buffer); - - new BundleValidator().validateJarFile(logger, new JarFile(JARS_DIR + "snapshot_bundle.jar")); + JarFile jarFile = createTemporaryJarFile("snapshot_bundle"); + new BundleValidator().validateJarFile(logger, jarFile); assertTrue(buffer.toString().contains("Deploying snapshot bundle")); } @@ -56,14 +61,50 @@ public class BundleValidatorTest { final StringBuffer buffer = new StringBuffer(); DeployLogger logger = createDeployLogger(buffer); BundleValidator validator = new BundleValidator(); - Manifest manifest = new Manifest(Files.newInputStream(Paths.get(JARS_DIR + "/manifest-producing-import-warnings.MF"))); - validator.validateManifest(logger, "my-app-bundle.jar", manifest); + JarFile jarFile = createTemporaryJarFile("import-warnings"); + validator.validateJarFile(logger, jarFile); assertThat(buffer.toString()) - .contains("For JAR file 'my-app-bundle.jar': \n" + + .contains("For JAR file 'import-warnings.jar': \n" + "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" + "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); } + @Test + public void outputs_deploy_warning_on_deprecated_dependency() throws IOException { + StringBuffer buffer = new StringBuffer(); + DeployLogger logger = createDeployLogger(buffer); + BundleValidator validator = new BundleValidator(); + JarFile jarFile = createTemporaryJarFile("pom-xml-warnings"); + validator.validateJarFile(logger, jarFile); + assertThat(buffer.toString()) + .contains("The pom.xml of bundle 'pom-xml-warnings.jar' includes a dependency to the artifact " + + "'com.yahoo.vespa:vespa-http-client-extensions'. \n" + + "This artifact will be removed in Vespa 8. " + + "Programmatic use can be safely removed from system/staging tests. " + + "See internal Vespa 8 release notes for details.\n"); + } + + private JarFile createTemporaryJarFile(String testArtifact) throws IOException { + Path jarFile = tempDir.newFile(testArtifact + ".jar").toPath(); + Path artifactDirectory = Paths.get("src/test/cfg/application/validation/testjars/" + testArtifact); + try (JarOutputStream out = new JarOutputStream(Files.newOutputStream(jarFile))) { + Files.walk(artifactDirectory).forEach(path -> { + Path relativePath = artifactDirectory.relativize(path); + String zipName = relativePath.toString(); + uncheck(() -> { + if (Files.isDirectory(path)) { + out.putNextEntry(new JarEntry(zipName + "/")); + } else { + out.putNextEntry(new JarEntry(zipName)); + out.write(Files.readAllBytes(path)); + } + out.closeEntry(); + }); + }); + } + return new JarFile(jarFile.toFile()); + } + private DeployLogger createDeployLogger(StringBuffer buffer) { return (__, message) -> buffer.append(message).append('\n'); } diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index d8459b175a7..a7f6a2a97a7 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -106,7 +106,7 @@ export LD_LIBRARY_PATH="$VESPA_HOME/lib64" case $1 in start) - nohup sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & + nohup nice sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS jvmopts="-Xms32M -Xmx128M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java index d3295c023b0..a96f9db855c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import java.time.Clock; import java.time.Duration; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index e4a0fa81f94..f8ee3e5e0c9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 42ccdffb2af..ab02594d164 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -578,7 +578,7 @@ public class SessionRepository { // Skip sessions newly added (we might have a session in the file system, but not in ZooKeeper, // we don't want to touch any of them) if (newSessions.contains(candidate.getSessionId())) { - log.log(Level.INFO, () -> "Skipping session " + candidate.getSessionId() + ", newly created: "); + log.log(Level.FINE, () -> "Skipping expiring newly created session " + candidate.getSessionId()); continue; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 3c762b7c2e5..4a19e8d16fb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationName; diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 97b5ff90e28..3ce00e5d08a 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -35,111 +35,99 @@ <scope>test</scope> </dependency> </dependencies> - <profiles> - <profile> - <id>enforce-container-deps</id> - <activation> - <activeByDefault>false</activeByDefault> - <property> - <!-- Dependency resolution is broken for old maven used in our CentOS docker containers --> - <name>maven.version</name> - <value>!3.0.5</value> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <executions> - <execution> - <!-- To allow running 'mvn enforcer:enforce' from the command line --> - <id>default-cli</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <bannedDependencies> - <excludes> - <!-- Only allow explicitly listed deps in provided and compile scope --> - <exclude>*:*:*:jar:provided:*</exclude> - <exclude>*:*:*:jar:compile:*</exclude> - </excludes> - <includes> - <include>com.yahoo.vespa</include> - <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <!-- To allow running 'mvn enforcer:enforce' from the command line --> + <id>default-cli</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <excludes> + <!-- Only allow explicitly listed deps in provided and compile scope --> + <exclude>*:*:*:jar:provided:*</exclude> + <exclude>*:*:*:jar:compile:*</exclude> + </excludes> + <includes> + <include>com.yahoo.vespa</include> + <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> - <!-- Use version range for jax deps, because jersey and junit affect the versions. --> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> - <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> - <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> - <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> - <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-logging:commons-logging:[1.2]:jar:provided</include> - <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> - <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> - <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> - <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> - <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> - <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> - <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> - <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> - <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:javax.inject:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-api:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-locator:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-utils:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:osgi-resource-locator:[${hk2.osgi-resource-locator.version}]:jar:provided</include> - <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-entity-filtering:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-proxy-client:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> - <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> - <include>org.json:json:[${org.json.version}]:jar:provided</include> - <include>org.jvnet.mimepull:mimepull:[${mimepull.version}]:jar:provided</include> - <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> - <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> - <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> - </includes> - </bannedDependencies> - </rules> - <fail>true</fail> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> + <!-- Use version range for jax deps, because jersey and junit affect the versions. --> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> + + <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> + <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> + <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> + <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> + <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> + <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> + <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> + <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> + <include>commons-logging:commons-logging:[1.2]:jar:provided</include> + <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> + <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> + <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> + <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> + <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> + <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> + <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> + <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> + <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> + <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> + <include>org.glassfish.hk2.external:aopalliance-repackaged:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2.external:javax.inject:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:hk2-api:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:hk2-locator:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:hk2-utils:[${hk2.version}]:jar:provided</include> + <include>org.glassfish.hk2:osgi-resource-locator:[${hk2.osgi-resource-locator.version}]:jar:provided</include> + <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-entity-filtering:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-proxy-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> + <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> + <include>org.json:json:[${org.json.version}]:jar:provided</include> + <include>org.jvnet.mimepull:mimepull:[${mimepull.version}]:jar:provided</include> + <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> + <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> + <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> + <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> + <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> + </includes> + </bannedDependencies> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> <properties> <maven.javadoc.skip>true</maven.javadoc.skip> diff --git a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def index e2856e137f0..c58f9944d61 100644 --- a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def +++ b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def @@ -24,7 +24,7 @@ jvm.stacksize int default=512 restart jvm.compressedClassSpaceSize int default=32 restart ## Base value of maximum direct memory size (in megabytes) -jvm.baseMaxDirectMemorySize int default=75 restart +jvm.baseMaxDirectMemorySize int default=16 restart ## Amount of direct memory used for caching. (in megabytes) jvm.directMemorySizeCache int default=0 restart diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index daf9cfd428f..2c087166e36 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -15,9 +15,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -73,7 +70,6 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag hideSharedRoutingEndpoint; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -81,7 +77,6 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); - this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -186,17 +181,21 @@ public class RoutingController { return EndpointList.copyOf(endpoints); } - /** Read and return zone-scoped endpoints for given deployments, grouped by their zone */ - public Map<ZoneId, List<Endpoint>> readZoneEndpointsOf(Collection<DeploymentId> deployments) { - var endpoints = new TreeMap<ZoneId, List<Endpoint>>(Comparator.comparing(ZoneId::value)); + /** Read test runner endpoints for given deployments, grouped by their zone */ + public Map<ZoneId, List<Endpoint>> readTestRunnerEndpointsOf(Collection<DeploymentId> deployments) { + TreeMap<ZoneId, List<Endpoint>> endpoints = new TreeMap<>(Comparator.comparing(ZoneId::value)); for (var deployment : deployments) { - EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone).not().legacy(); - zoneEndpoints = directEndpoints(zoneEndpoints, deployment.applicationId()); + EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone) + .not().legacy(); + EndpointList directEndpoints = zoneEndpoints.direct(); + if (!directEndpoints.isEmpty()) { + zoneEndpoints = directEndpoints; // Use only direct endpoints if we have any + } if ( ! zoneEndpoints.isEmpty()) { endpoints.put(deployment.zoneId(), zoneEndpoints.asList()); } } - return Collections.unmodifiableMap(endpoints); + return Collections.unmodifiableSortedMap(endpoints); } /** Returns certificate DNS names (CN and SAN values) for given deployment */ @@ -355,17 +354,6 @@ public class RoutingController { Priority.normal)); } - /** Returns direct routing endpoints if any exist and feature flag is set for given application */ - // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant - public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) { - boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - EndpointList directEndpoints = endpoints.direct(); - if (hideSharedEndpoint && !directEndpoints.isEmpty()) { - return directEndpoints; - } - return endpoints; - } - /** * Assigns one or more global rotations to given application, if eligible. The given application is implicitly * stored, ensuring that the assigned rotation(s) are persisted when this returns. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 684c497571d..225634634b2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -459,7 +459,7 @@ public class InternalStepRunner implements StepRunner { /** Returns true iff all calls to endpoint in the deployment give 100 consecutive 200 OK responses on /status.html. */ private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) { - var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zoneId))); + var endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(new DeploymentId(id, zoneId))); if ( ! endpoints.containsKey(zoneId)) return false; @@ -485,7 +485,7 @@ public class InternalStepRunner implements StepRunner { private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { DeploymentId deployment = new DeploymentId(id, zone); - Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readZoneEndpointsOf(Set.of(deployment)); + Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(deployment)); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; @@ -593,7 +593,7 @@ public class InternalStepRunner implements StepRunner { deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); - var endpoints = controller.routing().readZoneEndpointsOf(deployments); + var endpoints = controller.routing().readTestRunnerEndpointsOf(deployments); if ( ! endpoints.containsKey(zoneId)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 9f72b68372c..acaf35133d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.InstanceName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index caad32d9a17..9b1a03039fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.collections.Pair; import com.yahoo.component.Version; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 73a80a0a94a..d54c3c770ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -96,11 +96,11 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; -import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; -import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; @@ -139,7 +139,6 @@ import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.Scanner; @@ -1379,18 +1378,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { EndpointList zoneEndpoints = controller.routing().readEndpointsOf(deploymentId) .scope(Endpoint.Scope.zone); if (!legacyEndpoints) { - zoneEndpoints = zoneEndpoints.not().legacy(); + zoneEndpoints = zoneEndpoints.not().legacy().direct(); } - for (var endpoint : controller.routing().directEndpoints(zoneEndpoints, deploymentId.applicationId())) { + for (var endpoint : zoneEndpoints) { toSlime(endpoint, endpointArray.addObject()); } // Add declared endpoints EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application) .targets(deploymentId); if (!legacyEndpoints) { - declaredEndpoints = declaredEndpoints.not().legacy(); + declaredEndpoints = declaredEndpoints.not().legacy().direct(); } - for (var endpoint : controller.routing().directEndpoints(declaredEndpoints, deploymentId.applicationId())) { + for (var endpoint : declaredEndpoints) { toSlime(endpoint, endpointArray.addObject()); } @@ -2077,7 +2076,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(testConfigSerializer.configSlime(id, type, false, - controller.routing().readZoneEndpointsOf(deployments), + controller.routing().readTestRunnerEndpointsOf(deployments), controller.applications().reachableContentClustersByZone(deployments))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 226a7ca9561..7b66e60b426 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -35,6 +35,7 @@ import com.yahoo.yolean.Exceptions; import java.net.URI; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.logging.Level; @@ -98,22 +99,21 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private HttpResponse endpoints(Path path) { - var instanceId = instanceFrom(path); - var endpoints = controller.routing().readDeclaredEndpointsOf(instanceId) - .sortedBy(Comparator.comparing(Endpoint::name)) - .asList(); - - var deployments = endpoints.stream() - .flatMap(e -> e.deployments().stream()) - .distinct() - .sorted(Comparator.comparing(DeploymentId::dottedString)) - .collect(Collectors.toList()); - - var deploymentsStatus = deployments.stream() - .collect(Collectors.toMap( - deploymentId -> deploymentId, - deploymentId -> controller.routing().of(deploymentId).routingStatus()) - ); + ApplicationId instanceId = instanceFrom(path); + List<Endpoint> endpoints = controller.routing().readDeclaredEndpointsOf(instanceId) + .sortedBy(Comparator.comparing(Endpoint::dnsName)) + .asList(); + + List<DeploymentId> deployments = endpoints.stream() + .flatMap(e -> e.deployments().stream()) + .distinct() + .collect(Collectors.toList()); + + Map<DeploymentId, RoutingStatus> deploymentsStatus = deployments.stream() + .collect(Collectors.toMap( + deploymentId -> deploymentId, + deploymentId -> controller.routing().of(deploymentId).routingStatus()) + ); var slime = new Slime(); var root = slime.setObject(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index ce8c981fb5b..8ed09bfdf8a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -46,6 +46,7 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -333,12 +334,13 @@ public class ControllerTest { for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), Set.of("rotation-id-01", + "app1.tenant1.global.vespa.oath.cloud", "app1--tenant1.global.vespa.oath.cloud", "app1--tenant1.global.vespa.yahooapis.com"), tester.configServer().containerEndpointNames(context.deploymentIdIn(deployment.zone()))); } context.flushDnsUpdates(); - assertEquals(2, tester.controllerTester().nameService().records().size()); + assertEquals(3, tester.controllerTester().nameService().records().size()); Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); @@ -352,9 +354,11 @@ public class ControllerTest { List<String> globalDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.instanceId()) .scope(Endpoint.Scope.global) + .sortedBy(Comparator.comparing(Endpoint::dnsName)) .mapToList(Endpoint::dnsName); assertEquals(List.of("app1--tenant1.global.vespa.oath.cloud", - "app1--tenant1.global.vespa.yahooapis.com"), + "app1--tenant1.global.vespa.yahooapis.com", + "app1.tenant1.global.vespa.oath.cloud"), globalDnsNames); } @@ -375,11 +379,11 @@ public class ControllerTest { assertFalse(deployments.isEmpty()); var notWest = Set.of( - "rotation-id-01", "foobar--app1--tenant1.global.vespa.oath.cloud", - "rotation-id-02", "app1--tenant1.global.vespa.oath.cloud", - "rotation-id-03", "all--app1--tenant1.global.vespa.oath.cloud" + "rotation-id-01", "foobar--app1--tenant1.global.vespa.oath.cloud", "foobar.app1.tenant1.global.vespa.oath.cloud", + "rotation-id-02", "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", + "rotation-id-03", "all--app1--tenant1.global.vespa.oath.cloud", "all.app1.tenant1.global.vespa.oath.cloud" ); - var west = Sets.union(notWest, Set.of("rotation-id-04", "west--app1--tenant1.global.vespa.oath.cloud")); + var west = Sets.union(notWest, Set.of("rotation-id-04", "west--app1--tenant1.global.vespa.oath.cloud", "west.app1.tenant1.global.vespa.oath.cloud")); for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), @@ -388,7 +392,7 @@ public class ControllerTest { } context.flushDnsUpdates(); - assertEquals(4, tester.controllerTester().nameService().records().size()); + assertEquals(8, tester.controllerTester().nameService().records().size()); var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record1.isPresent()); @@ -430,7 +434,7 @@ public class ControllerTest { for (var zone : List.of(west, central)) { assertEquals( "Zone " + zone + " is a member of global endpoint", - Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } @@ -448,13 +452,13 @@ public class ControllerTest { for (var zone : List.of(west, central)) { assertEquals( "Zone " + zone + " is a member of global endpoint", - Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } assertEquals( "Zone " + east + " is a member of global endpoint", - Set.of("rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud"), + Set.of("rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud", "east.app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(east)) ); @@ -471,9 +475,9 @@ public class ControllerTest { assertEquals( "Zone " + zone + " is a member of global endpoint", zone.equals(east) - ? Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", - "rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud") - : Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + ? Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", + "rotation-id-02", "east--app1--tenant1.global.vespa.oath.cloud", "east.app1.tenant1.global.vespa.oath.cloud") + : Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud"), tester.configServer().containerEndpointNames(context.deploymentIdIn(zone)) ); } @@ -565,7 +569,7 @@ public class ControllerTest { .build(); context.submit(applicationPackage).deploy(); - assertEquals(1, tester.controllerTester().nameService().records().size()); + assertEquals(2, tester.controllerTester().nameService().records().size()); { Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); @@ -608,7 +612,7 @@ public class ControllerTest { .region("us-central-1") .build(); context.submit(applicationPackage).deploy(); - assertEquals(1, tester.controllerTester().nameService().records().size()); + assertEquals(2, tester.controllerTester().nameService().records().size()); var record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); assertTrue(record.isPresent()); @@ -628,7 +632,7 @@ public class ControllerTest { assertEquals("rotation-id-02", context.instance().rotations().get(0).rotationId().asString()); // DNS records are created for the newly assigned rotation - assertEquals(2, tester.controllerTester().nameService().records().size()); + assertEquals(4, tester.controllerTester().nameService().records().size()); var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record1.isPresent()); @@ -986,6 +990,7 @@ public class ControllerTest { var zone1 = ZoneId.from("prod", "us-west-1"); var zone2 = ZoneId.from("prod", "us-east-3"); var applicationPackageBuilder = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .region(zone1.region()) .region(zone2.region()); tester.controllerTester().zoneRegistry() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 91a12d3b465..a338efd856c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -59,7 +59,7 @@ public class ApplicationPackageBuilder { private String upgradePolicy = null; private String upgradeRollout = null; private String globalServiceId = null; - private String athenzIdentityAttributes = null; + private String athenzIdentityAttributes = "athenz-domain='domain' athenz-service='service'"; private String searchDefinition = "search test { }"; private boolean explicitSystemTest = false; private boolean explicitStagingTest = false; @@ -195,7 +195,12 @@ public class ApplicationPackageBuilder { public ApplicationPackageBuilder athenzIdentity(AthenzDomain domain, AthenzService service) { this.athenzIdentityAttributes = Text.format("athenz-domain='%s' athenz-service='%s'", domain.value(), - service.value()); + service.value()); + return this; + } + + public ApplicationPackageBuilder withoutAthenzIdentity() { + this.athenzIdentityAttributes = null; return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index fd7ba8693e2..c7c434910f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1161,7 +1161,7 @@ public class DeploymentTriggerTest { @Test public void testsInSeparateInstance() { String deploymentSpec = - "<deployment version='1.0'>\n" + + "<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + " <instance id='canary'>\n" + " <upgrade policy='canary' />\n" + " <test />\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 23ab91aaf8c..97cfd520e67 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -52,26 +52,28 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry */ public ZoneRegistryMock(SystemName system) { this.system = system; - this.zones = system.isPublic() ? - List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-eu-west-1a")) : - List.of(ZoneApiMock.fromId("test.us-east-1"), - ZoneApiMock.fromId("staging.us-east-3"), - ZoneApiMock.fromId("dev.us-east-1"), - ZoneApiMock.fromId("dev.aws-us-east-2a"), - ZoneApiMock.fromId("perf.us-east-3"), - ZoneApiMock.fromId("prod.aws-us-east-1a"), - ZoneApiMock.fromId("prod.ap-northeast-1"), - ZoneApiMock.fromId("prod.ap-northeast-2"), - ZoneApiMock.fromId("prod.ap-southeast-1"), - ZoneApiMock.fromId("prod.us-east-3"), - ZoneApiMock.fromId("prod.us-west-1"), - ZoneApiMock.fromId("prod.us-central-1"), - ZoneApiMock.fromId("prod.eu-west-1")); - // All zones use a shared routing method by default - setRoutingMethod(this.zones, system.isPublic() ? RoutingMethod.exclusive : RoutingMethod.shared); + if (system.isPublic()) { + this.zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-eu-west-1a")); + setRoutingMethod(this.zones, RoutingMethod.exclusive); + } else { + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), + ZoneApiMock.fromId("dev.us-east-1"), + ZoneApiMock.fromId("dev.aws-us-east-2a"), + ZoneApiMock.fromId("perf.us-east-3"), + ZoneApiMock.fromId("prod.aws-us-east-1a"), + ZoneApiMock.fromId("prod.ap-northeast-1"), + ZoneApiMock.fromId("prod.ap-northeast-2"), + ZoneApiMock.fromId("prod.ap-southeast-1"), + ZoneApiMock.fromId("prod.us-east-3"), + ZoneApiMock.fromId("prod.us-west-1"), + ZoneApiMock.fromId("prod.us-central-1"), + ZoneApiMock.fromId("prod.eu-west-1")); + setRoutingMethod(this.zones, RoutingMethod.sharedLayer4, RoutingMethod.shared); + } } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 581ec68b3dd..d97f1d58043 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -283,7 +283,7 @@ public class MetricsReporterTest { context.submit(applicationPackage).deploy(); reporter.maintain(); - assertEquals("Deployment queues name services requests", 4, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); + assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); context.flushDnsUpdates(); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index a4de6ab7700..92055c85a53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -238,7 +238,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Test public void create_application_on_deploy() { var application = ApplicationName.from("unique"); - var applicationPackage = new ApplicationPackageBuilder().build(); + var applicationPackage = new ApplicationPackageBuilder().withoutAthenzIdentity().build(); assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); @@ -256,6 +256,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { var application = ApplicationName.from("unique"); var applicationPackage = new ApplicationPackageBuilder() .trustDefaultCertificate() + .withoutAthenzIdentity() .build(); assertTrue(tester.controller().applications().getApplication(TenantAndApplicationId.from(tenantName, application)).isEmpty()); @@ -273,6 +274,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { private ApplicationPackageBuilder prodBuilder() { return new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("default") .region("aws-us-east-1c"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index c78f83ced57..df8db83f229 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -26,8 +26,6 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; @@ -126,6 +124,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final String accessDenied = "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}"; private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("default") .globalServiceId("foo") .region("us-central-1") @@ -135,6 +134,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); private static final ApplicationPackage applicationPackageInstance1 = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("instance1") .globalServiceId("foo") .region("us-central-1") @@ -338,6 +338,7 @@ public class ApplicationApiTest extends ControllerContainerTest { app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsCentral1); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("instance1") .globalServiceId("foo") .region("us-west-1") @@ -816,6 +817,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Sixth attempt has a multi-instance deployment spec, and is accepted. ApplicationPackage multiInstanceSpec = new ApplicationPackageBuilder() + .withoutAthenzIdentity() .instances("instance1,instance2") .region("us-central-1") .parallel("us-west-1", "us-east-3") @@ -1522,7 +1524,7 @@ public class ApplicationApiTest extends ControllerContainerTest { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), - List.of(RoutingMethod.exclusive, RoutingMethod.shared)); + RoutingMethod.exclusive); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) .instances("instance1") @@ -1541,15 +1543,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), new File("deployment-with-routing-policy.json")); - // GET deployment including legacy endpoints - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/instance1", GET) - .userIdentity(USER_ID) - .properties(Map.of("includeLegacyEndpoints", "true")), - new File("deployment-with-routing-policy-legacy.json")); - - // Hide shared endpoints - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.HIDE_SHARED_ROUTING_ENDPOINT.id(), true); - // GET deployment tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/instance1", GET) .userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json deleted file mode 100644 index 4955c549b4b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "instance1", - "environment": "prod", - "region": "us-west-1", - "endpoints": [ - { - "cluster": "default", - "tls": true, - "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/", - "scope": "zone", - "routingMethod": "exclusive", - "legacy": false - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": false - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa.yahooapis.com:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": true - } - ], - "clusters": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", - "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/?recursive=true&application=tenant1.application1.instance1", - "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", - "version": "6.1.0", - "revision": "1.0.1-commit1", - "deployTimeEpochMs": "(ignore)", - "screwdriverId": "1000", - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1", - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "quota": "(ignore)", - "activity": {}, - "metrics": { - "queriesPerSecond": 0.0, - "writesPerSecond": 0.0, - "documentCount": 0.0, - "queryLatencyMillis": 0.0, - "writeLatencyMillis": 0.0 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 97ac87fb5a0..4457bede34e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -12,14 +12,6 @@ "scope": "zone", "routingMethod": "exclusive", "legacy": false - }, - { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": false } ], "clusters":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/clusters", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index ab2a3bf945c..a1c32b67eb0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -6,22 +6,6 @@ "region": "us-central-1", "endpoints": [ { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": false - }, - { - "cluster": "foo", - "tls": true, - "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", - "scope": "global", - "routingMethod": "shared", - "legacy": false - }, - { "cluster": "foo", "tls": true, "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index 45df6aad67c..175c45eb2cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -11,7 +11,7 @@ { "at": 0, "type": "info", - "message": " |-- https://application--tenant.us-east-1.dev.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://application.tenant.us-east-1.dev.vespa.oath.cloud/ (cluster 'default')" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index f2f8e14f093..2ca520c0122 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -8,9 +8,9 @@ { "cluster": "default", "tls": true, - "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/", + "url": "https://instance1.application1.tenant1.us-east-1.dev.vespa.oath.cloud/", "scope": "zone", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 62ad3a2db7e..25d306ae764 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -9,22 +9,6 @@ "region": "us-central-1", "endpoints": [ { - "cluster": "default", - "tls": true, - "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", - "scope": "zone", - "routingMethod": "shared", - "legacy": false - }, - { - "cluster": "foo", - "tls": true, - "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", - "scope": "global", - "routingMethod": "shared", - "legacy": false - }, - { "cluster": "foo", "tls": true, "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index 7f59eaf75c2..6c6092cbd88 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -232,7 +232,7 @@ { "at": 0, "type": "info", - "message": " |-- https://application--tenant.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://application.tenant.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { "at": 0, @@ -259,7 +259,7 @@ { "at": 0, "type": "info", - "message": " |-- https://application--tenant.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" + "message": " |-- https://application.tenant.us-east-1.test.vespa.oath.cloud/ (cluster 'default')" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json index 0632ab7a67b..047e10b16e4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json @@ -5,7 +5,7 @@ "isCI": false, "endpoints": { "dev.us-east-1": [ - "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" + "https://my-user.application1.tenant1.us-east-1.dev.vespa.oath.cloud/" ], "prod.us-central-1": [ "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" @@ -13,7 +13,7 @@ }, "zoneEndpoints": { "dev.us-east-1": { - "default": "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" + "default": "https://my-user.application1.tenant1.us-east-1.dev.vespa.oath.cloud/" }, "prod.us-central-1": { "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index 5b2fabcaff8..0b128ebf7a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -258,7 +258,7 @@ public class RoutingApiTest extends ControllerContainerTest { // One shared and one exclusive zone deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone), - RoutingMethod.shared); + RoutingMethod.sharedLayer4); deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(eastZone), RoutingMethod.exclusive); @@ -273,7 +273,7 @@ public class RoutingApiTest extends ControllerContainerTest { .build(); context.submit(applicationPackage).deploy(); - // GET status for zone using shared routing + // GET status for zone using sharedLayer4 routing tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/deployment-status-initial.json")); @@ -331,4 +331,5 @@ public class RoutingApiTest extends ControllerContainerTest { tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/endpoint", "", Request.Method.GET), new File("endpoint/endpoints.json")); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json index f78f913cb7e..7804b277e52 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/endpoint/endpoints.json @@ -26,6 +26,33 @@ "changedAt": 1497618757000 } ] + }, + { + "name": "default", + "dnsName": "a1.t1.global.vespa.oath.cloud", + "routingMethod": "sharedLayer4", + "cluster": "default", + "scope": "global", + "zones": [ + { + "routingMethod": "sharedLayer4", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": 1497618757000 + }, + { + "routingMethod": "sharedLayer4", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": 1497618757000 + } + ] } ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index 9a56123e8e3..9be95dc7a79 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -60,8 +60,8 @@ public class RotationRepositoryTest { Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); - assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); + assertEquals(URI.create("https://app1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).direct().first().get().url()); try (RotationLock lock = repository.lock()) { List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), application.instance(), @@ -175,10 +175,10 @@ public class RotationRepositoryTest { var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance1.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).direct().first().get().url()); + assertEquals(URI.create("https://instance2.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).direct().first().get().url()); } @Test @@ -197,10 +197,10 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance1.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).direct().first().get().url()); + assertEquals(URI.create("https://instance2.application1.tenant1.global.vespa.oath.cloud/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).direct().first().get().url()); } private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 065636604d9..c94bfec948e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -248,18 +248,11 @@ public class Flags { public static final UnboundBooleanFlag DELETE_UNMAINTAINED_CERTIFICATES = defineFeatureFlag( "delete-unmaintained-certificates", false, - List.of("andreer"), "2021-09-23", "2022-02-01", + List.of("andreer"), "2021-09-23", "2022-02-14", "Whether to delete certificates that are known by provider but not by controller", "Takes effect on next run of EndpointCertificateMaintainer" ); - public static final UnboundBooleanFlag USE_NEW_ENDPOINT_CERTIFICATE_PROVIDER_URL = defineFeatureFlag( - "use-new-endpoint-certificate-provider-url", true, - List.of("andreer"), "2021-12-14", "2022-01-14", - "Use the new URL for the endpoint certificate provider API", - "Takes effect immediately" - ); - public static final UnboundBooleanFlag ENABLE_TENANT_DEVELOPER_ROLE = defineFeatureFlag( "enable-tenant-developer-role", false, List.of("bjorncs"), "2021-09-23", "2022-02-01", @@ -361,7 +354,7 @@ public class Flags { public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( "fail-deployment-with-invalid-jvm-options", false, - List.of("hmusum"), "2021-12-20", "2022-01-20", + List.of("hmusum"), "2021-12-20", "2022-02-20", "Whether to fail deployments with invalid JVM options in services.xml", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index e7c364cb7de..7a29ba88d46 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -182,6 +182,25 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-no-log4j</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <!-- Fail validation for apps with log4j deps in compile or provided scope. --> + <excludes> + <exclude>log4j:log4j:*:jar:compile</exclude> + <exclude>log4j:log4j:*:jar:provided</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:compile</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:provided</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> </executions> </plugin> diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java index 8b35935f0fa..bd498dc02df 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JaxRsTimeouts.java @@ -10,14 +10,14 @@ public interface JaxRsTimeouts { /** * The connect timeout, which must be at least 1ms. Called once per real REST call. * - * Throws com.google.common.util.concurrent.UncheckedTimeoutException on timeout. + * @throws com.yahoo.concurrent.UncheckedTimeoutException on timeout. */ Duration getConnectTimeoutOrThrow(); /** * The read timeout, which must be at least 1ms. Called once per real REST call. * - * Throws com.google.common.util.concurrent.UncheckedTimeoutException on timeout. + * @throws com.yahoo.concurrent.UncheckedTimeoutException on timeout. */ Duration getReadTimeoutOrThrow(); } diff --git a/logd/src/apps/retention/retention-enforcer.sh b/logd/src/apps/retention/retention-enforcer.sh index 6355600ee4a..24bc61e5764 100755 --- a/logd/src/apps/retention/retention-enforcer.sh +++ b/logd/src/apps/retention/retention-enforcer.sh @@ -64,6 +64,7 @@ mark_pid() { } check_pidfile() { + [ -f $PIDF ] || return 0 read pid < $PIDF [ "$pid" = $$ ] && return 0 if [ "$pid" ] && [ $pid -gt $$ ]; then @@ -105,8 +106,17 @@ maybe_collect() { process_file() { dbfile="$1" now=$(date +%s) + dbf_ts_prefix=${dbfile##*.} + dbf_ts_beg=${dbf_ts_prefix}00000 + dbf_ts_end=${dbf_ts_prefix}99999 + add=$((86400 * $RETAIN_DAYS)) + earliest_expire=$((${dbf_ts_beg} + $add)) + if [ $earliest_expire -gt $now ]; then + return 0 + fi found=0 while read timestamp logfilename; do + sleep 1 for fn in $logfilename $logfilename.*z*; do if [ -f "$fn" ]; then found=1 @@ -115,8 +125,7 @@ process_file() { done done < $dbfile if [ $found = 0 ]; then - ts=${dbfile##*.}99999 - maybe_collect "$ts" "$dbfile" + maybe_collect "${dbf_ts_end}" "$dbfile" fi } @@ -124,6 +133,7 @@ process_all() { for dbf in $DBDIR/logfiles.* ; do [ -f "$dbf" ] || continue process_file "$dbf" + sleep 1 done } @@ -139,5 +149,6 @@ mainloop() { # MAIN: prepare_stuff +sleep 600 mainloop exit 0 diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java index 12934c23926..af167fda5c6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java @@ -15,7 +15,7 @@ import java.util.Set; public class AddNode { public final String hostname; - public final Optional<String> id; + public final String id; public final Optional<String> parentHostname; public final Optional<String> nodeFlavor; public final Optional<FlavorOverrides> flavorOverrides; @@ -24,15 +24,15 @@ public class AddNode { public final Set<String> ipAddresses; public final Set<String> additionalIpAddresses; - public static AddNode forHost(String hostname, Optional<String> id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { + public static AddNode forHost(String hostname, String id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { return new AddNode(hostname, id, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses); } - public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { - return new AddNode(hostname, Optional.empty(), Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); + public static AddNode forNode(String hostname, String id, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { + return new AddNode(hostname, id, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); } - private AddNode(String hostname, Optional<String> id, Optional<String> parentHostname, + private AddNode(String hostname, String id, Optional<String> parentHostname, Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<NodeResources> nodeResources, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index acb6ece6fc1..6e31e699e2c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -26,7 +26,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; public class NodeSpec { private final String hostname; - private final Optional<String> id; + private final String id; private final NodeState state; private final NodeType type; private final String flavor; @@ -72,7 +72,7 @@ public class NodeSpec { public NodeSpec( String hostname, - Optional<String> id, + String id, Optional<DockerImage> wantedDockerImage, Optional<DockerImage> currentDockerImage, NodeState state, @@ -148,8 +148,8 @@ public class NodeSpec { return hostname; } - /** Returns the cloud-specific ID of the host. */ - public Optional<String> id() { + /** Returns unique node ID */ + public String id() { return id; } @@ -406,7 +406,7 @@ public class NodeSpec { public static class Builder { private String hostname; - private Optional<String> id = Optional.empty(); + private String id; private NodeState state; private NodeType type; private String flavor; @@ -441,6 +441,7 @@ public class NodeSpec { public Builder(NodeSpec node) { hostname(node.hostname); + id(node.id); state(node.state); type(node.type); flavor(node.flavor); @@ -477,7 +478,7 @@ public class NodeSpec { } public Builder id(String id) { - this.id = Optional.of(id); + this.id = id; return this; } @@ -786,6 +787,7 @@ public class NodeSpec { */ public static Builder testSpec(String hostname, NodeState state) { Builder builder = new Builder() + .id(hostname) .hostname(hostname) .state(state) .type(NodeType.tenant) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 38e725360a0..b99a3bb84d7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -162,7 +162,7 @@ public class RealNodeRepository implements NodeRepository { return new NodeSpec( node.hostname, - Optional.ofNullable(node.id), + node.id, Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString), Optional.ofNullable(node.currentDockerImage).map(DockerImage::fromString), nodeState, @@ -244,7 +244,7 @@ public class RealNodeRepository implements NodeRepository { private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) { NodeRepositoryNode node = new NodeRepositoryNode(); - node.id = addNode.id.orElse("fake-" + addNode.hostname); + node.id = addNode.id; node.hostname = addNode.hostname; node.parentHostname = addNode.parentHostname.orElse(null); addNode.nodeFlavor.ifPresent(f -> node.flavor = f); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 3d7a3b73ccd..dff72fe81f1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -79,7 +79,7 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts) { Map<String, NodeAgentContext> nodeAgentContextsByHostname = nodeAgentContexts.stream() - .collect(Collectors.toMap(NodeAdminImpl::nodeAgentId, Function.identity())); + .collect(Collectors.toMap(ctx -> ctx.node().id(), Function.identity())); // Stop and remove NodeAgents that should no longer be running diff(nodeAgentWithSchedulerByHostname.keySet(), nodeAgentContextsByHostname.keySet()) @@ -222,14 +222,4 @@ public class NodeAdminImpl implements NodeAdmin { NodeAgent nodeAgent = nodeAgentFactory.create(contextManager, context); return new NodeAgentWithScheduler(nodeAgent, contextManager); } - - private static String nodeAgentId(NodeAgentContext nac) { - // NodeAgentImpl has some internal state that should not be reused when the same hostname is re-allocated - // to a different application/cluster, solve this by including reservation timestamp in the key. - return nac.hostname().value() + "-" + nac.node().events().stream() - .filter(event -> "reserved".equals(event.type())) - .findFirst() - .map(event -> Long.toString(event.at().toEpochMilli())) - .orElse(""); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java deleted file mode 100644 index 553f2ffba36..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/Templar.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; - -/** - * A very simple template engine when there's little complexity and lots of Velocity special characters $ and #, - * i.e. typically shell script. - * - * @author hakonhall - */ -public class Templar { - private final String template; - - private static final String prefix = "<%="; - private static final String suffix = "%>"; - - private final Map<String, String> settings = new HashMap<>(); - - public static Templar fromUtf8File(Path path) { - return new Templar(new UnixPath(path).readUtf8File()); - } - - public Templar(String template) { - this.template = template; - } - - public Templar set(String name, String value) { - settings.put(name, value); - return this; - } - - public String resolve() { - StringBuilder text = new StringBuilder(template.length() * 2); - - int start= 0; - int end; - - for (; start < template.length(); start = end) { - int prefixStart = template.indexOf(prefix, start); - - - if (prefixStart == -1) { - text.append(template, start, template.length()); - break; - } else { - text.append(template, start, prefixStart); - } - - int suffixStart = template.indexOf(suffix, prefixStart + prefix.length()); - if (suffixStart == -1) { - throw new IllegalArgumentException("Prefix at offset " + prefixStart + " is not terminated"); - } - - int prefixEnd = prefixStart + prefix.length(); - String name = template.substring(prefixEnd, suffixStart).trim(); - String value = settings.get(name); - if (value == null) { - throw new IllegalArgumentException("No value is set for name '" + name + "' at offset " + prefixEnd); - } - - text.append(value); - - end = suffixStart + suffix.length(); - } - - return text.toString(); - } - - public FileWriter getFileWriterTo(Path path) { - return new FileWriter(path, this::resolve); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java index 4a00115cee4..fdb14189656 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java @@ -18,7 +18,7 @@ class IfSection extends Section { IfSection(CursorRange range, boolean negated, String name, Cursor nameOffset, SectionList ifSections, Optional<SectionList> elseSections) { - super(range); + super("if", range); this.negated = negated; this.name = name; this.nameOffset = nameOffset; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java index 831dc3fe5e8..a137a27f3bb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java @@ -17,7 +17,7 @@ class ListSection extends Section { private final List<Template> elements = new ArrayList<>(); ListSection(CursorRange range, String name, Cursor nameOffset, Template body) { - super(range); + super("list", range); this.name = name; this.nameOffset = new Cursor(nameOffset); this.body = body; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java index c03653253af..667c515dcec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; */ class LiteralSection extends Section { LiteralSection(CursorRange range) { - super(range); + super("literal", range); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java index dd92af14609..0869b7f181c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java @@ -1,22 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.template; -import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; -import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; - /** * @author hakonhall */ public class NameAlreadyExistsTemplateException extends TemplateException { - public NameAlreadyExistsTemplateException(String name, CursorRange range) { - super("Name '" + name + "' already exists in the " + describeSection(range)); - } - - public NameAlreadyExistsTemplateException(String name, Cursor firstNameLocation, - Cursor secondNameLocation) { - super("Section named '" + name + "' at " + - firstNameLocation.calculateLocation().lineAndColumnText() + - " conflicts with earlier section with the same name at " + - secondNameLocation.calculateLocation().lineAndColumnText()); + public NameAlreadyExistsTemplateException(String name, Section first, Section second) { + super("The name '" + name + "' of the " + second.type() + " section at " + + second.range().start().calculateLocation().lineAndColumnText() + + " is in conflict with the identically named " + first.type() + " section at " + + first.range().start().calculateLocation().lineAndColumnText()); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java index 2c52fd5c34e..42927cd2c04 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java @@ -12,10 +12,12 @@ import java.util.Objects; * @author hakonhall */ abstract class Section { + private final String type; private final CursorRange range; private Template template; - protected Section(CursorRange range) { + protected Section(String type, CursorRange range) { + this.type = type; this.range = range; } @@ -24,6 +26,7 @@ abstract class Section { /** Guaranteed to return non-null after TemplateBuilder::build() returns. */ protected Template template() { return Objects.requireNonNull(template); } + protected String type() { return type; } protected CursorRange range() { return range; } abstract void appendTo(StringBuilder buffer); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java index 41e8c3e65ce..6d80ac2cad9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.template; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileWriter; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; @@ -25,12 +26,15 @@ import java.util.Optional; * id: a valid Java identifier * </pre> * + * <p>Other directive delimiters than "%{" and "}" may be used, see {@link TemplateDescriptor}.</p> + * * <p>Fill the template with variable values ({@link #set(String, String) set()}, set if conditions * ({@link #set(String, boolean)}), add list elements ({@link #add(String) add()}, etc, and finally * render it as a String ({@link #render()}).</p> * * <p>To reuse a template, create the template and work on snapshots of that ({@link #snapshot()}).</p> * + * @see TemplateDescriptor * @author hakonhall */ public class Template implements Form { @@ -85,6 +89,11 @@ public class Template implements Form { return template; } + public FileWriter getFileWriterTo(Path path) { + String content = render(); + return new FileWriter(path, () -> content); + } + /** Must be called (if there is a parent) before any other method. */ void setParent(Template parent) { this.parent = parent; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java index 8041a17fe74..2827a9eb005 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java @@ -35,8 +35,7 @@ class TemplateBuilder { ListSection existing = lists.get(section.name()); if (existing != null) - throw new NameAlreadyExistsTemplateException(section.name(), existing.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), existing, section); sampleVariables.put(section.name(), section); allSections.add(section); @@ -48,8 +47,7 @@ class TemplateBuilder { ListSection list = lists.get(section.name()); if (list != null) - throw new NameAlreadyExistsTemplateException(section.name(), list.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), list, section); sampleIfSections.put(section.name(), section); allSections.add(section); @@ -58,18 +56,15 @@ class TemplateBuilder { void addListSection(ListSection section) { VariableSection variableSection = sampleVariables.get(section.name()); if (variableSection != null) - throw new NameAlreadyExistsTemplateException(section.name(), variableSection.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), variableSection, section); IfSection ifSection = sampleIfSections.get(section.name()); if (ifSection != null) - throw new NameAlreadyExistsTemplateException(section.name(), ifSection.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), ifSection, section); ListSection previous = lists.put(section.name(), section); if (previous != null) - throw new NameAlreadyExistsTemplateException(section.name(), previous.nameOffset(), - section.nameOffset()); + throw new NameAlreadyExistsTemplateException(section.name(), previous, section); allSections.add(section); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java index c2202dea4a0..41fd716a3e6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java @@ -47,9 +47,7 @@ class TemplateParser { if (current.eot()) { if (!sentinels.contains(Sentinel.EOT)) { - throw new BadTemplateException(current, - "Missing end directive for section started at " + - start.calculateLocation().lineAndColumnText()); + throw new BadTemplateException(start, "Missing end directive for section started"); } return Sentinel.EOT; } @@ -71,12 +69,12 @@ class TemplateParser { switch (type) { case "else": if (!sentinels.contains(Sentinel.ELSE)) - throw new BadTemplateException(startOfType, "Extraneous 'else'"); + throw new BadTemplateException(startOfType, "Stray 'else'"); parseEndDirective(); return Optional.of(Sentinel.ELSE); case "end": if (!sentinels.contains(Sentinel.END)) - throw new BadTemplateException(startOfType, "Extraneous 'end'"); + throw new BadTemplateException(startOfType, "Stray 'end'"); parseEndDirective(); return Optional.of(Sentinel.END); case "if": diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java index 6a7bec2e485..17fedce55fa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java @@ -15,7 +15,7 @@ class VariableSection extends Section { private final Cursor nameOffset; VariableSection(CursorRange range, String name, Cursor nameOffset) { - super(range); + super("variable", range); this.name = name; this.nameOffset = nameOffset; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 5d15d4353e2..3256b16a6c5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -161,20 +161,20 @@ public class RealNodeRepositoryTest { @Test public void testAddNodes() { AddNode host = AddNode.forHost("host123.domain.tld", - Optional.of("id1"), + "id1", "default", Optional.of(FlavorOverrides.ofDisk(123)), NodeType.confighost, Set.of("::1"), Set.of("::2", "::3")); NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); - AddNode node = AddNode.forNode("host123-1.domain.tld", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); + AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); nodeRepositoryApi.addNodes(List.of(host, node)); NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); - assertEquals("id1", hostSpec.id().orElseThrow()); + assertEquals("id1", hostSpec.id()); assertEquals("default", hostSpec.flavor()); assertEquals(123, hostSpec.diskGb(), 0); assertEquals(NodeType.confighost, hostSpec.type()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplarTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplarTest.java deleted file mode 100644 index ed410ffc1d1..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TemplarTest.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author hakonhall - */ -public class TemplarTest { - @Test - public void test() { - Templar templar = new Templar("x y <%= foo %>, some other <%=bar%> text"); - templar.set("foo", "fidelity") - .set("bar", "halimov") - .set("not", "used"); - - assertEquals("x y fidelity, some other halimov text", templar.resolve()); - } -}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java index e010b9780c6..b0260af1582 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java @@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * @author hakonhall @@ -147,6 +148,70 @@ public class TemplateTest { template.render()); } + @Test + void badTemplates() { + assertException(BadTemplateException.class, "Unknown section 'zoo' at line 2 and column 6", + () -> Template.from("foo\nbar%{zoo}")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", + () -> Template.from("%{=")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 4", + () -> Template.from("%{=¬atoken}")); + + assertException(BadTemplateException.class, "Expected identifier at line 1 and column 8", + () -> Template.from("%{list ¬atoken}")); + + assertException(BadTemplateException.class, "Missing end directive for section started at line 1 and column 12", + () -> Template.from("%{list foo}missing end")); + + assertException(BadTemplateException.class, "Stray 'end' at line 1 and column 3", + () -> Template.from("%{end}stray end")); + + assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 4 has not been set: notset", + () -> Template.from("%{=notset}").render()); + + assertException(TemplateNameNotSetException.class, "Variable at line 1 and column 6 has not been set: cond", + () -> Template.from("%{if cond}%{end}").render()); + + assertException(NotBooleanValueTemplateException.class, "cond was set to a non-boolean value: must be true or false", + () -> Template.from("%{if cond}%{end}").set("cond", 1).render()); + + assertException(NoSuchNameTemplateException.class, "No such element 'listname' in the template section starting at " + + "line 1 and column 1, and ending at line 1 and column 4", + () -> Template.from("foo").add("listname")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 6 is in conflict with the identically " + + "named variable section at line 1 and column 1", + () -> Template.from("%{=a}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the variable section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{=a}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the list section at line 1 and column 14 is in conflict with the identically " + + "named if section at line 1 and column 1", + () -> Template.from("%{if a}%{end}%{list a}%{end}")); + + assertException(NameAlreadyExistsTemplateException.class, + "The name 'a' of the if section at line 1 and column 16 is in conflict with the identically " + + "named list section at line 1 and column 1", + () -> Template.from("%{list a}%{end}%{if a}%{end}")); + } + + private <T extends Throwable> void assertException(Class<T> class_, String message, Runnable runnable) { + T exception = assertThrows(class_, runnable::run); + assertEquals(message, exception.getMessage()); + } + private Template getTemplate(String filename) { return Template.at(Path.of("src/test/resources/" + filename)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index a7ba8b27851..55c225c3dad 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index ba28d8e6b9a..379bb2566df 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.component.Version; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 2a9cdb7e0eb..ddaec86b340 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; @@ -13,7 +13,6 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.orchestrator.config.OrchestratorConfig; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java index 20b27ea7632..fd62f2b4b70 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClient.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.orchestrator.OrchestratorContext; import java.io.IOException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 7c8be703310..f4929a4f09e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.jaxrs.client.JaxRsStrategy; import com.yahoo.vespa.orchestrator.OrchestratorContext; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java index 141687bd269..cdae68a0d06 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeouts.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.jaxrs.client.JaxRsTimeouts; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index fc790089517..d393117d57c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.model; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import java.util.logging.Level; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; @@ -22,6 +21,7 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.logging.Level; import java.util.logging.Logger; public class StorageNodeImpl implements StorageNode { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java index c56866fdad6..e1cf7989a91 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandler.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.jdisc.Response; import com.yahoo.restapi.JacksonJsonResponse; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java index 33391e42a93..05df303e3e7 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandler.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.jdisc.Response; import com.yahoo.restapi.JacksonJsonResponse; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java index 00ab1a964c8..ce4dd5ad4ef 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestratorContext; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index a29c186d30c..ce68dca0cc5 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java index 970a8682c5a..6f22ff74ad8 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientTimeoutsTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.test.ManualClock; import com.yahoo.time.TimeBudget; import org.junit.Before; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java index ea7566f9d51..03f2c0b5e3b 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.HttpRequestBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Metric; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java index 089a2dc8709..9641e045431 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionRequestHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.container.jdisc.HttpRequestBuilder; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.RestApiTestDriver; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java index 07c8662f656..20bfb09a53d 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.status; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestratorContext; diff --git a/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp b/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp index 313f3e9a270..c0d94ba6376 100644 --- a/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/resource_usage_tracker/resource_usage_tracker_test.cpp @@ -57,9 +57,10 @@ public: ~ResourceUsageTrackerTest(); - void notify(double disk_usage, double memory_usage) + void notify(double disk_usage, double memory_usage, double transient_disk_usage = 0.0, double transient_memory_usage = 0.0) { - _notifier.notify(DiskMemUsageState({ 0.8, disk_usage }, { 0.8, memory_usage })); + _notifier.notify(DiskMemUsageState({ 0.8, disk_usage }, { 0.8, memory_usage }, + transient_disk_usage, transient_memory_usage)); } ResourceUsage get_usage() { return _listener->get_usage(); } @@ -77,6 +78,15 @@ TEST_F(ResourceUsageTrackerTest, resource_usage_is_forwarded_to_listener) EXPECT_EQ(ResourceUsage(0.75, 0.25), get_usage()); } +TEST_F(ResourceUsageTrackerTest, transient_resource_usage_is_subtracted_from_absolute_usage) +{ + auto register_guard = _tracker->set_listener(*_listener); + notify(0.8, 0.5, 0.4, 0.2); + EXPECT_EQ(ResourceUsage(0.4, 0.3), get_usage()); + notify(0.8, 0.5, 0.9, 0.6); + EXPECT_EQ(ResourceUsage(0.0, 0.0), get_usage()); +} + TEST_F(ResourceUsageTrackerTest, forwarding_depends_on_register_guard) { auto register_guard = _tracker->set_listener(*_listener); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp index 6307604598d..63e8b1d5196 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp @@ -94,8 +94,13 @@ void ResourceUsageTracker::notifyDiskMemUsage(DiskMemUsageState state) { std::lock_guard guard(_lock); - // TODO: Subtract transient resource (memory and disk) usage from the absolute numbers. - _resource_usage = ResourceUsage(state.diskState().usage(), state.memoryState().usage(), _resource_usage.get_attribute_address_space_usage()); + // The transient resource usage is subtracted from the absolute resource usage + // before it eventually is reported to the cluster controller (to decide whether to block client feed). + // This ensures that the transient resource usage is covered by the resource headroom on the content node, + // instead of leading to feed blocked due to natural fluctuations. + double adj_disk_usage = std::max(0.0, state.diskState().usage() - state.transient_disk_usage()); + double adj_memory_usage = std::max(0.0, state.memoryState().usage() - state.transient_memory_usage()); + _resource_usage = ResourceUsage(adj_disk_usage, adj_memory_usage, _resource_usage.get_attribute_address_space_usage()); if (_listener != nullptr) { _listener->update_resource_usage(_resource_usage); } diff --git a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index dd71380f64a..56352ff3c0d 100644 --- a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp @@ -30,6 +30,28 @@ TEST("test that all tasks are executed") { EXPECT_EQUAL(10000u, counter); } +TEST("test that executor can overflow") { + constexpr size_t NUM_TASKS = 1000; + std::atomic<uint64_t> counter(0); + vespalib::Gate gate; + SingleExecutor executor(sequenced_executor, 10, false, 1, 1ms); + executor.execute(makeLambdaTask([&gate] { gate.await();})); + + for(size_t i(0); i < NUM_TASKS; i++) { + executor.execute(makeLambdaTask([&counter, i] { + EXPECT_EQUAL(i, counter); + counter++; + })); + } + EXPECT_EQUAL(0u, counter); + ExecutorStats stats = executor.getStats(); + EXPECT_EQUAL(NUM_TASKS + 1, stats.acceptedTasks); + EXPECT_EQUAL(NUM_TASKS, stats.queueSize.max()); + gate.countDown(); + executor.sync(); + EXPECT_EQUAL(NUM_TASKS, counter); +} + void verifyResizeTaskLimit(bool up) { std::mutex lock; std::condition_variable cond; @@ -38,7 +60,7 @@ void verifyResizeTaskLimit(bool up) { constexpr uint32_t INITIAL = 20; const uint32_t INITIAL_2inN = roundUp2inN(INITIAL); double waterMarkRatio = 0.5; - SingleExecutor executor(sequenced_executor, INITIAL, INITIAL*waterMarkRatio, 10ms); + SingleExecutor executor(sequenced_executor, INITIAL, true, INITIAL*waterMarkRatio, 10ms); EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); EXPECT_EQUAL(uint32_t(INITIAL_2inN*waterMarkRatio), executor.get_watermark()); diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index 76b0235301b..58ae862f7c6 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -67,7 +67,7 @@ SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint3 for (uint32_t id = 0; id < threads; ++id) { if (optimize == OptimizeFor::THROUGHPUT) { uint32_t watermark = (kindOfWatermark == 0) ? taskLimit / 10 : kindOfWatermark; - executors.push_back(std::make_unique<SingleExecutor>(func, taskLimit, watermark, 100ms)); + executors.push_back(std::make_unique<SingleExecutor>(func, taskLimit, true, watermark, 100ms)); } else { executors.push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit, func)); } diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp index a99bce0a705..21ed90c3d22 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp @@ -7,12 +7,12 @@ namespace vespalib { SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit) - : SingleExecutor(func, taskLimit, taskLimit/10, 100ms) + : SingleExecutor(func, taskLimit, true, taskLimit/10, 100ms) { } -SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit, uint32_t watermark, duration reactionTime) - : _watermarkRatio(watermark < taskLimit ? double(watermark) / taskLimit : 1.0), - _taskLimit(vespalib::roundUp2inN(taskLimit)), +SingleExecutor::SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime) + : _watermarkRatio(watermark < reservedQueueSize ? double(watermark) / reservedQueueSize : 1.0), + _taskLimit(vespalib::roundUp2inN(reservedQueueSize)), _wantedTaskLimit(_taskLimit.load()), _rp(0), _tasks(std::make_unique<Task::UP[]>(_taskLimit)), @@ -30,9 +30,13 @@ SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit, uint32_t wat _wp(0), _watermark(_taskLimit.load()*_watermarkRatio), _reactionTime(reactionTime), - _closed(false) + _closed(false), + _overflow() { - assert(taskLimit >= watermark); + assert(reservedQueueSize >= watermark); + if ( ! isQueueSizeHard) { + _overflow = std::make_unique<ArrayQueue<Task::UP>>(); + } _thread.start(); } @@ -62,10 +66,12 @@ SingleExecutor::execute(Task::UP task) { if (_closed) { return task; } - wait_for_room(guard); - wp = _wp.load(std::memory_order_relaxed); - _tasks[index(wp)] = std::move(task); - _wp.store(wp + 1, std::memory_order_release); + task = wait_for_room_or_put_in_overflow_Q(guard, std::move(task)); + if (task) { + wp = move_to_main_q(guard, std::move(task)); + } else { + wp = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(guard); + } } if (wp == _wakeupConsumerAt.load(std::memory_order_relaxed)) { _consumerCondition.notify_one(); @@ -73,6 +79,24 @@ SingleExecutor::execute(Task::UP task) { return task; } +uint64_t +SingleExecutor::numTasks() { + if (_overflow) { + Lock guard(_mutex); + return num_tasks_in_main_q() + num_tasks_in_overflow_q(guard); + } else { + return num_tasks_in_main_q(); + } +} + +uint64_t +SingleExecutor::move_to_main_q(Lock &, Task::UP task) { + uint64_t wp = _wp.load(std::memory_order_relaxed); + _tasks[index(wp)] = std::move(task); + _wp.store(wp + 1, std::memory_order_release); + return wp; +} + void SingleExecutor::setTaskLimit(uint32_t taskLimit) { _wantedTaskLimit = vespalib::roundUp2inN(taskLimit); @@ -81,7 +105,7 @@ SingleExecutor::setTaskLimit(uint32_t taskLimit) { void SingleExecutor::drain(Lock & lock) { uint64_t wp = _wp.load(std::memory_order_relaxed); - while (numTasks() > 0) { + while (numTasks(lock) > 0) { _consumerCondition.notify_one(); sleepProducer(lock, 100us, wp); } @@ -97,7 +121,7 @@ SingleExecutor::wakeup() { SingleExecutor & SingleExecutor::sync() { Lock lock(_mutex); - uint64_t wp = _wp.load(std::memory_order_relaxed); + uint64_t wp = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(lock); while (wp > _rp.load(std::memory_order_acquire)) { _consumerCondition.notify_one(); sleepProducer(lock, 100us, wp); @@ -119,7 +143,7 @@ SingleExecutor::run() { _producerCondition.notify_all(); _wakeupConsumerAt.store(_wp.load(std::memory_order_relaxed) + get_watermark(), std::memory_order_relaxed); Lock lock(_mutex); - if (numTasks() <= 0) { + if (numTasks(lock) <= 0) { steady_time now = steady_clock::now(); _threadIdleTracker.set_idle(now); _consumerCondition.wait_until(lock, now + _reactionTime); @@ -134,6 +158,22 @@ void SingleExecutor::drain_tasks() { while (numTasks() > 0) { run_tasks_till(_wp.load(std::memory_order_acquire)); + move_overflow_to_main_q(); + } +} + +void +SingleExecutor::move_overflow_to_main_q() +{ + if ( ! _overflow) return; + Lock guard(_mutex); + move_overflow_to_main_q(guard); +} +void +SingleExecutor::move_overflow_to_main_q(Lock & guard) { + while ( !_overflow->empty() && num_tasks_in_main_q() < _taskLimit.load(std::memory_order_relaxed)) { + move_to_main_q(guard, std::move(_overflow->front())); + _overflow->pop(); } } @@ -151,26 +191,42 @@ SingleExecutor::run_tasks_till(uint64_t available) { } } -void -SingleExecutor::wait_for_room(Lock & lock) { +Executor::Task::UP +SingleExecutor::wait_for_room_or_put_in_overflow_Q(Lock & guard, Task::UP task) { uint64_t wp = _wp.load(std::memory_order_relaxed); uint64_t taskLimit = _taskLimit.load(std::memory_order_relaxed); if (taskLimit != _wantedTaskLimit.load(std::memory_order_relaxed)) { - drain(lock); + drain(guard); _tasks = std::make_unique<Task::UP[]>(_wantedTaskLimit); _taskLimit = _wantedTaskLimit.load(); _watermark = _taskLimit * _watermarkRatio; } - _queueSize.add(numTasks()); - while (numTasks() >= _taskLimit.load(std::memory_order_relaxed)) { - sleepProducer(lock, _reactionTime, wp - get_watermark()); + uint64_t numTaskInQ = numTasks(guard); + _queueSize.add(numTaskInQ); + if (numTaskInQ >= _taskLimit.load(std::memory_order_relaxed)) { + if (_overflow) { + _overflow->push(std::move(task)); + } else { + while (numTasks(guard) >= _taskLimit.load(std::memory_order_relaxed)) { + sleepProducer(guard, _reactionTime, wp - get_watermark()); + } + } + } else { + if (_overflow && !_overflow->empty()) { + _overflow->push(std::move(task)); + } + } + if (_overflow && !_overflow->empty()) { + assert(!task); + move_overflow_to_main_q(guard); } + return task; } ExecutorStats SingleExecutor::getStats() { Lock lock(_mutex); - uint64_t accepted = _wp.load(std::memory_order_relaxed); + uint64_t accepted = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(lock); steady_time now = steady_clock::now(); _idleTracker.was_idle(_threadIdleTracker.reset(now)); ExecutorStats stats(_queueSize, (accepted - _lastAccepted), 0, _wakeupCount); diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h index e76e3f17a41..4fdc217e701 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/threadexecutor.h> #include <vespa/vespalib/util/thread.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/executor_idle_tracking.h> #include <thread> #include <atomic> @@ -19,8 +20,8 @@ namespace vespalib { */ class SingleExecutor final : public vespalib::SyncableThreadExecutor, vespalib::Runnable { public: - SingleExecutor(init_fun_t func, uint32_t taskLimit); - SingleExecutor(init_fun_t func, uint32_t taskLimit, uint32_t watermark, duration reactionTime); + SingleExecutor(init_fun_t func, uint32_t reservedQueueSize); + SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime); ~SingleExecutor() override; Task::UP execute(Task::UP task) override; void setTaskLimit(uint32_t taskLimit) override; @@ -39,12 +40,22 @@ private: void drain_tasks(); void sleepProducer(Lock & guard, duration maxWaitTime, uint64_t wakeupAt); void run_tasks_till(uint64_t available); - void wait_for_room(Lock & guard); + Task::UP wait_for_room_or_put_in_overflow_Q(Lock & guard, Task::UP task); + uint64_t move_to_main_q(Lock & guard, Task::UP task); + void move_overflow_to_main_q(); + void move_overflow_to_main_q(Lock & guard); uint64_t index(uint64_t counter) const { return counter & (_taskLimit.load(std::memory_order_relaxed) - 1); } - uint64_t numTasks() const { + uint64_t numTasks(); + uint64_t numTasks(Lock & guard) const { + return num_tasks_in_main_q() + num_tasks_in_overflow_q(guard); + } + uint64_t num_tasks_in_overflow_q(Lock &) const { + return _overflow ? _overflow->size() : 0; + } + uint64_t num_tasks_in_main_q() const { return _wp.load(std::memory_order_relaxed) - _rp.load(std::memory_order_acquire); } const double _watermarkRatio; @@ -67,6 +78,7 @@ private: std::atomic<uint32_t> _watermark; const duration _reactionTime; bool _closed; + std::unique_ptr<ArrayQueue<Task::UP>> _overflow; }; } diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index 3d24ee87879..5dafd9c5eda 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -113,7 +113,7 @@ class UnrevertableRemoveEntryProcessor : public BucketProcessor::EntryProcessor public: using DocumentIdsAndTimeStamps = std::vector<std::pair<spi::Timestamp, spi::DocumentId>>; UnrevertableRemoveEntryProcessor(DocumentIdsAndTimeStamps & to_remove) - : _to_remove(to_remove) + : _to_remove(to_remove) {} void process(spi::DocEntry& entry) override { diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index f4923bf79f1..cf1f65aa05a 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -366,6 +366,25 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-no-log4j</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <!-- Fail validation for apps with log4j deps in compile or provided scope. --> + <excludes> + <exclude>log4j:log4j:*:jar:compile</exclude> + <exclude>log4j:log4j:*:jar:provided</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:compile</exclude> + <exclude>org.apache.logging.log4j:log4j-core:(,2.17.0]:jar:provided</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> </executions> </plugin> diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh index c4e70c362b0..fc99a68614a 100755 --- a/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client-standalone.sh @@ -4,6 +4,5 @@ exec java \ -Djava.awt.headless=true \ -Xms128m -Xmx2048m \ ---add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.util.logging.config.file=`dirname $0`/logging.properties \ -cp `dirname $0`/vespa-feed-client-cli-jar-with-dependencies.jar ai.vespa.feed.client.impl.CliClient "$@" diff --git a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh index 7dbdc056524..6e9aefd56c7 100755 --- a/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh +++ b/vespa-feed-client-cli/src/main/sh/vespa-feed-client.sh @@ -79,6 +79,5 @@ exec java \ -Djava.library.path=${VESPA_HOME}/libexec64/native:${VESPA_HOME}/lib64 \ -Djava.awt.headless=true \ -Xms128m -Xmx2048m $(getJavaOptionsIPV46) \ ---add-opens=java.base/sun.security.ssl=ALL-UNNAMED \ -Djava.util.logging.config.file=${VESPA_HOME}/conf/vespa-feed-client/logging.properties \ -cp ${VESPA_HOME}/lib/jars/vespa-feed-client-cli-jar-with-dependencies.jar ai.vespa.feed.client.impl.CliClient "$@" diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java b/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java index 44bcec4f0eb..7fa5ecfcdee 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; -import com.google.common.util.concurrent.UncheckedTimeoutException; - import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/UncheckedTimeoutException.java b/vespajlib/src/main/java/com/yahoo/concurrent/UncheckedTimeoutException.java new file mode 100644 index 00000000000..b9fa0c6cb3c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/UncheckedTimeoutException.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.concurrent.TimeoutException; + +/** + * Unchecked alternative for {@link java.util.concurrent.TimeoutException}. + * + * @author bjorncs + */ +public class UncheckedTimeoutException extends RuntimeException { + + public UncheckedTimeoutException() {} + + public UncheckedTimeoutException(TimeoutException cause) { super(cause.getMessage(), cause); } + + public UncheckedTimeoutException(String message) { super(message); } + + public UncheckedTimeoutException(Throwable cause) { super(cause); } + + public UncheckedTimeoutException(String message, Throwable cause) { super(message, cause); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index da22dbdc336..1edf8e4edbe 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.net.HostName; import java.math.BigDecimal; diff --git a/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java b/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java index b18a0f397f6..64cf9dd522c 100644 --- a/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java +++ b/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.time; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import java.time.Clock; import java.time.Duration; diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java index 7f2f0deea66..604c29e7289 100644 --- a/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java +++ b/vespajlib/src/test/java/com/yahoo/concurrent/maintenance/MaintainerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent.maintenance; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import org.junit.Test; import java.time.Duration; diff --git a/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java b/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java index bad95883df9..c296137dfb6 100644 --- a/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java +++ b/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.time; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.test.ManualClock; import org.junit.Test; diff --git a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp index 73e70e7fd89..8f3dd8ab006 100644 --- a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp +++ b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp @@ -2,11 +2,11 @@ #pragma once -#include <stdint.h> -#include <stdlib.h> +#include "traits.h" +#include <cstdint> +#include <cstdlib> #include <cassert> #include <algorithm> -#include "traits.h" namespace vespalib { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java index 4b0ea433cdd..56ae65bb317 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.stats.LockStats; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java index 40bfb70109d..4cb510e7904 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.mock; -import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.collections.Pair; import com.yahoo.concurrent.Lock; import com.yahoo.concurrent.Locks; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; diff --git a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java index 607ab4004a7..e5a1ea7c683 100644 --- a/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java +++ b/zkfacade/src/test/java/com/yahoo/vespa/curator/stats/LockTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.stats; -import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.vespa.curator.Lock; import org.apache.curator.framework.recipes.locks.InterProcessLock; import org.junit.Before; |