diff options
11 files changed, 27 insertions, 275 deletions
diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java index 35a95ed3d89..a39896928c8 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java @@ -1,19 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin; -import com.yahoo.osgi.maven.ProjectBundleClassPaths; import com.yahoo.vespa.config.VespaVersion; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; import java.util.Enumeration; import java.util.jar.Attributes; import java.util.jar.JarEntry; @@ -22,13 +18,7 @@ import java.util.jar.Manifest; import java.util.regex.Pattern; import java.util.zip.ZipEntry; -import static com.yahoo.osgi.maven.ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.endsWith; -import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -152,33 +142,4 @@ public class BundleTest { assertThat(webInfUrl, containsString("/WEB-INF/web.xml")); } - // TODO Vespa 8: Remove, the classpath mappings file is only needed for jersey resources to work in the application test framework. - // When this test is removed, also remove the maven-resources-plugin from the 'main' test bundle's pom. - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - @SuppressWarnings("unchecked") - @Test - public void bundle_class_path_mappings_are_generated() throws Exception { - ZipEntry classpathMappingsEntry = jarFile.getEntry(CLASSPATH_MAPPINGS_FILENAME); - - assertNotNull( - "Could not find " + CLASSPATH_MAPPINGS_FILENAME + " in the test bundle", - classpathMappingsEntry); - - Path mappingsFile = tempFolder.newFile(CLASSPATH_MAPPINGS_FILENAME).toPath(); - Files.copy(jarFile.getInputStream(classpathMappingsEntry), mappingsFile, REPLACE_EXISTING); - - ProjectBundleClassPaths bundleClassPaths = ProjectBundleClassPaths.load(mappingsFile); - - assertThat(bundleClassPaths.mainBundle.bundleSymbolicName, is("main")); - - Collection<String> mainBundleClassPaths = bundleClassPaths.mainBundle.classPathElements; - - assertThat(mainBundleClassPaths, - hasItems( - endsWith("target/classes"), - anyOf( - allOf(containsString("/jrt-"), containsString(".jar")), - containsString("/jrt.jar")))); - } } diff --git a/bundle-plugin-test/test-bundles/main/pom.xml b/bundle-plugin-test/test-bundles/main/pom.xml index 190e1c9d90f..c315318453e 100644 --- a/bundle-plugin-test/test-bundles/main/pom.xml +++ b/bundle-plugin-test/test-bundles/main/pom.xml @@ -42,32 +42,6 @@ <WebInfUrl>/WEB-INF/web.xml</WebInfUrl> </configuration> </plugin> - <plugin> - <!-- Trick to copy bundle-plugin.bundle-classpath-mappings.json from target/test-classes to target/classes --> - <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-bundle-classpath-mappings-from-test-to-main</id> - <!-- NOTE: Must be done after generating classpath-mappings and before assembling the bundle (see the test-bundles pom) --> - <phase>process-test-sources</phase> - <goals> - <goal>copy-resources</goal> - </goals> - <configuration> - <outputDirectory>${project.build.outputDirectory}</outputDirectory> - <overwrite>true</overwrite> - <resources> - <resource> - <directory>${project.build.testOutputDirectory}</directory> - <includes> - <include>bundle-plugin.bundle-classpath-mappings.json</include> - </includes> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/bundle-plugin-test/test-bundles/pom.xml b/bundle-plugin-test/test-bundles/pom.xml index 712ccb5542e..ab19b3578c5 100644 --- a/bundle-plugin-test/test-bundles/pom.xml +++ b/bundle-plugin-test/test-bundles/pom.xml @@ -30,13 +30,6 @@ <version>${project.version}</version> <executions> <execution> - <id>generate-classpath-mappings</id> - <phase>generate-test-sources</phase> - <goals> - <goal>generate-bundle-classpath-mappings</goal> - </goals> - </execution> - <execution> <id>package-test-bundles</id> <!-- Must be done after generating classpath-mappings and copying it in the 'main' bundle. --> <phase>test-compile</phase> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java deleted file mode 100644 index e94e05512aa..00000000000 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.mojo; - -import com.google.common.base.Preconditions; -import com.yahoo.container.plugin.bundle.AnalyzeBundle; -import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths; -import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths.BundleClasspathMapping; -import com.yahoo.container.plugin.util.Artifacts; -import org.apache.maven.artifact.Artifact; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugins.annotations.Mojo; -import org.apache.maven.plugins.annotations.Parameter; -import org.apache.maven.plugins.annotations.ResolutionScope; -import org.apache.maven.project.MavenProject; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Generates mapping from Bundle-SymbolicName to classpath elements, e.g myBundle -> [.m2/repository/com/mylib/Mylib.jar, - * myBundleProject/target/classes] The mapping in stored in a json file. - * - * @author Tony Vaagenes - * @author ollivir - */ -@Mojo(name = "generate-bundle-classpath-mappings", requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME, threadSafe = true) -public class GenerateBundleClassPathMappingsMojo extends AbstractMojo { - @Parameter(defaultValue = "${project}") - private MavenProject project = null; - - //TODO: Combine with com.yahoo.container.plugin.mojo.GenerateOsgiManifestMojo.bundleSymbolicName - @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") - private String bundleSymbolicName = null; - - /* Sample output -- target/test-classes/bundle-plugin.bundle-classpath-mappings.json - { - "mainBundle": { - "bundleSymbolicName": "bundle-plugin-test", - "classPathElements": [ - "/Users/tonyv/Repos/vespa/bundle-plugin-test/target/classes", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/7-SNAPSHOT/jrt-7-SNAPSHOT.jar", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/annotations/7-SNAPSHOT/annotations-7-SNAPSHOT.jar" - ] - }, - "providedDependencies": [ - { - "bundleSymbolicName": "jrt", - "classPathElements": [ - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/7-SNAPSHOT/jrt-7-SNAPSHOT.jar" - ] - } - ] - } - */ - @Override - public void execute() throws MojoExecutionException { - Preconditions.checkNotNull(bundleSymbolicName); - - Artifacts.ArtifactSet artifacts = Artifacts.getArtifacts(project); - List<Artifact> embeddedArtifacts = artifacts.getJarArtifactsToInclude(); - List<Artifact> providedJarArtifacts = artifacts.getJarArtifactsProvided(); - - List<File> embeddedArtifactsFiles = embeddedArtifacts.stream().map(Artifact::getFile).collect(Collectors.toList()); - - List<String> classPathElements = Stream.concat(Stream.of(outputDirectory()), embeddedArtifactsFiles.stream()) - .map(File::getAbsolutePath).collect(Collectors.toList()); - - ProjectBundleClassPaths classPathMappings = new ProjectBundleClassPaths( - new BundleClasspathMapping(bundleSymbolicName, classPathElements), - providedJarArtifacts.stream().map(f -> createDependencyClasspathMapping(f)).filter(Optional::isPresent).map(Optional::get) - .collect(Collectors.toList())); - - try { - ProjectBundleClassPaths.save(testOutputPath().resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME), classPathMappings); - } catch (IOException e) { - throw new MojoExecutionException("Error saving to file " + testOutputPath(), e); - } - } - - private File outputDirectory() { - return new File(project.getBuild().getOutputDirectory()); - } - - private Path testOutputPath() { - return Paths.get(project.getBuild().getTestOutputDirectory()); - } - - /* TODO: - * 1) add the dependencies of the artifact in the future(i.e. dependencies of dependencies) - * or - * 2) obtain bundles with embedded dependencies from the maven repository, - * and support loading classes from the nested jar files in those bundles. - */ - Optional<BundleClasspathMapping> createDependencyClasspathMapping(Artifact artifact) { - return bundleSymbolicNameForArtifact(artifact) - .map(name -> new BundleClasspathMapping(name, Arrays.asList(artifact.getFile().getAbsolutePath()))); - } - - private static Optional<String> bundleSymbolicNameForArtifact(Artifact artifact) { - if (artifact.getFile().getName().endsWith(".jar")) { - return AnalyzeBundle.bundleSymbolicName(artifact.getFile()); - } else { - // Not the best heuristic. The other alternatives are parsing the pom file or - // storing information in target/classes when building the provided bundles. - return Optional.of(artifact.getArtifactId()); - } - } -} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ProjectBundleClassPaths.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ProjectBundleClassPaths.java deleted file mode 100644 index 42033f6ac73..00000000000 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ProjectBundleClassPaths.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Objects; - -/** - * Represents the bundles in a maven project and the classpath elements - * corresponding to code that would end up in the bundle. - * - * @author Tony Vaagenes - * @author bjorncs - */ -public class ProjectBundleClassPaths { - public static final String CLASSPATH_MAPPINGS_FILENAME = "bundle-plugin.bundle-classpath-mappings.json"; - - private static final ObjectMapper mapper = new ObjectMapper(); - - @JsonProperty("mainBundle") public final BundleClasspathMapping mainBundle; - @JsonProperty("providedDependencies") public final List<BundleClasspathMapping> providedDependencies; - - @JsonCreator - public ProjectBundleClassPaths(@JsonProperty("mainBundle") BundleClasspathMapping mainBundle, - @JsonProperty("providedDependencies") List<BundleClasspathMapping> providedDependencies) { - this.mainBundle = mainBundle; - this.providedDependencies = providedDependencies; - } - - public static void save(Path path, ProjectBundleClassPaths mappings) throws IOException { - Files.createDirectories(path.getParent()); - mapper.writeValue(path.toFile(), mappings); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ProjectBundleClassPaths that = (ProjectBundleClassPaths) o; - return Objects.equals(mainBundle, that.mainBundle) && - Objects.equals(providedDependencies, that.providedDependencies); - } - - @Override - public int hashCode() { - return Objects.hash(mainBundle, providedDependencies); - } - - public static class BundleClasspathMapping { - @JsonProperty("bundleSymbolicName") public final String bundleSymbolicName; - @JsonProperty("classPathElements") public final List<String> classPathElements; - - @JsonCreator - public BundleClasspathMapping(@JsonProperty("bundleSymbolicName") String bundleSymbolicName, - @JsonProperty("classPathElements") List<String> classPathElements) { - this.bundleSymbolicName = bundleSymbolicName; - this.classPathElements = classPathElements; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BundleClasspathMapping that = (BundleClasspathMapping) o; - return Objects.equals(bundleSymbolicName, that.bundleSymbolicName) && - Objects.equals(classPathElements, that.classPathElements); - } - - @Override - public int hashCode() { - return Objects.hash(bundleSymbolicName, classPathElements); - } - } -} diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 481211390e0..922a17fb71a 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -30,7 +30,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.41.v20210516</jetty.version> + <jetty.version>9.4.43.v20210629</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <junit5.version>5.7.0</junit5.version> <junit5.platform.version>1.7.0</junit5.platform.version> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 56fd6a64305..1eb27a3224a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -107,7 +107,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 27ee040fe4f..1e025bd8e35 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -208,6 +208,22 @@ public class SessionRepository { return List.copyOf(localSessionCache.values()); } + public Set<LocalSession> getLocalSessionsFromFileSystem() { + File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); + if (sessions == null) return Set.of(); + + Set<LocalSession> sessionIds = new HashSet<>(); + for (File session : sessions) { + long sessionId = Long.parseLong(session.getName()); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + LocalSession localSession = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient); + sessionIds.add(localSession); + } + return sessionIds; + } + private void loadLocalSessions(ExecutorService executor) { File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); if (sessions == null) return; @@ -553,7 +569,7 @@ public class SessionRepository { log.log(Level.FINE, () -> "Purging old sessions for tenant '" + tenantName + "'"); Set<LocalSession> toDelete = new HashSet<>(); try { - for (LocalSession candidate : getLocalSessions()) { + for (LocalSession candidate : getLocalSessionsFromFileSystem()) { Instant createTime = candidate.getCreateTime(); log.log(Level.FINE, () -> "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 1b31d02d222..8f8005894e5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -465,12 +465,13 @@ public class ApplicationRepositoryTest { // Create a local session without any data in zookeeper (corner case seen in production occasionally) // and check that expiring local sessions still work int sessionId = 6; - Files.createDirectory(new TenantFileSystemDirs(serverdb, tenant1).getUserApplicationDir(sessionId).toPath()); + TenantName tenantName = tester.tenant().getName(); + Files.createDirectory(new TenantFileSystemDirs(serverdb, tenantName).getUserApplicationDir(sessionId).toPath()); LocalSession localSession2 = new LocalSession(tenant1, sessionId, FilesApplicationPackage.fromFile(testApp), new SessionZooKeeperClient(curator, - tenant1, + tenantName, sessionId, ConfigUtils.getCanonicalHostName())); sessionRepository.addLocalSession(localSession2); diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 29e9f85256d..0e64737bb00 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -380,7 +380,7 @@ <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.41.v20210516</jetty.version> + <jetty.version>9.4.43.v20210629</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <org.lz4.version>1.7.1</org.lz4.version> <org.json.version>20090211</org.json.version> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index f8c390edb3b..157cdea831e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -28,7 +28,11 @@ import java.util.stream.Collectors; */ public class NameServiceForwarder { - private static final int QUEUE_CAPACITY = 300; + /** + * The number of {@link NameServiceRequest}s we allow to be queued. When the queue overflows, the first requests + * are dropped in a FIFO order until the queue shrinks below this capacity. + */ + private static final int QUEUE_CAPACITY = 400; private static final Logger log = Logger.getLogger(NameServiceForwarder.class.getName()); |