diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-07-16 08:38:32 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2021-07-16 08:38:32 +0200 |
commit | 3896b1f35f445b0b4859c6f93b01871936c09542 (patch) | |
tree | 549d5b4cac16a2143dccdadebf9398cd0a4aa493 | |
parent | 4c992d9f26189abd36796c342606c3cea4c990ea (diff) | |
parent | 073c5be0df2970bd4e6d926a56d14657965992d3 (diff) |
Merge branch 'master' into hmusum/remove-flag-2
95 files changed, 1479 insertions, 2063 deletions
diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java index 35a95ed3d89..a39896928c8 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java @@ -1,19 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin; -import com.yahoo.osgi.maven.ProjectBundleClassPaths; import com.yahoo.vespa.config.VespaVersion; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; import java.util.Enumeration; import java.util.jar.Attributes; import java.util.jar.JarEntry; @@ -22,13 +18,7 @@ import java.util.jar.Manifest; import java.util.regex.Pattern; import java.util.zip.ZipEntry; -import static com.yahoo.osgi.maven.ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.endsWith; -import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -152,33 +142,4 @@ public class BundleTest { assertThat(webInfUrl, containsString("/WEB-INF/web.xml")); } - // TODO Vespa 8: Remove, the classpath mappings file is only needed for jersey resources to work in the application test framework. - // When this test is removed, also remove the maven-resources-plugin from the 'main' test bundle's pom. - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - @SuppressWarnings("unchecked") - @Test - public void bundle_class_path_mappings_are_generated() throws Exception { - ZipEntry classpathMappingsEntry = jarFile.getEntry(CLASSPATH_MAPPINGS_FILENAME); - - assertNotNull( - "Could not find " + CLASSPATH_MAPPINGS_FILENAME + " in the test bundle", - classpathMappingsEntry); - - Path mappingsFile = tempFolder.newFile(CLASSPATH_MAPPINGS_FILENAME).toPath(); - Files.copy(jarFile.getInputStream(classpathMappingsEntry), mappingsFile, REPLACE_EXISTING); - - ProjectBundleClassPaths bundleClassPaths = ProjectBundleClassPaths.load(mappingsFile); - - assertThat(bundleClassPaths.mainBundle.bundleSymbolicName, is("main")); - - Collection<String> mainBundleClassPaths = bundleClassPaths.mainBundle.classPathElements; - - assertThat(mainBundleClassPaths, - hasItems( - endsWith("target/classes"), - anyOf( - allOf(containsString("/jrt-"), containsString(".jar")), - containsString("/jrt.jar")))); - } } diff --git a/bundle-plugin-test/test-bundles/main/pom.xml b/bundle-plugin-test/test-bundles/main/pom.xml index 190e1c9d90f..c315318453e 100644 --- a/bundle-plugin-test/test-bundles/main/pom.xml +++ b/bundle-plugin-test/test-bundles/main/pom.xml @@ -42,32 +42,6 @@ <WebInfUrl>/WEB-INF/web.xml</WebInfUrl> </configuration> </plugin> - <plugin> - <!-- Trick to copy bundle-plugin.bundle-classpath-mappings.json from target/test-classes to target/classes --> - <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-bundle-classpath-mappings-from-test-to-main</id> - <!-- NOTE: Must be done after generating classpath-mappings and before assembling the bundle (see the test-bundles pom) --> - <phase>process-test-sources</phase> - <goals> - <goal>copy-resources</goal> - </goals> - <configuration> - <outputDirectory>${project.build.outputDirectory}</outputDirectory> - <overwrite>true</overwrite> - <resources> - <resource> - <directory>${project.build.testOutputDirectory}</directory> - <includes> - <include>bundle-plugin.bundle-classpath-mappings.json</include> - </includes> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> </project> diff --git a/bundle-plugin-test/test-bundles/pom.xml b/bundle-plugin-test/test-bundles/pom.xml index 712ccb5542e..ab19b3578c5 100644 --- a/bundle-plugin-test/test-bundles/pom.xml +++ b/bundle-plugin-test/test-bundles/pom.xml @@ -30,13 +30,6 @@ <version>${project.version}</version> <executions> <execution> - <id>generate-classpath-mappings</id> - <phase>generate-test-sources</phase> - <goals> - <goal>generate-bundle-classpath-mappings</goal> - </goals> - </execution> - <execution> <id>package-test-bundles</id> <!-- Must be done after generating classpath-mappings and copying it in the 'main' bundle. --> <phase>test-compile</phase> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java deleted file mode 100644 index e94e05512aa..00000000000 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.mojo; - -import com.google.common.base.Preconditions; -import com.yahoo.container.plugin.bundle.AnalyzeBundle; -import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths; -import com.yahoo.container.plugin.osgi.ProjectBundleClassPaths.BundleClasspathMapping; -import com.yahoo.container.plugin.util.Artifacts; -import org.apache.maven.artifact.Artifact; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugins.annotations.Mojo; -import org.apache.maven.plugins.annotations.Parameter; -import org.apache.maven.plugins.annotations.ResolutionScope; -import org.apache.maven.project.MavenProject; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Generates mapping from Bundle-SymbolicName to classpath elements, e.g myBundle -> [.m2/repository/com/mylib/Mylib.jar, - * myBundleProject/target/classes] The mapping in stored in a json file. - * - * @author Tony Vaagenes - * @author ollivir - */ -@Mojo(name = "generate-bundle-classpath-mappings", requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME, threadSafe = true) -public class GenerateBundleClassPathMappingsMojo extends AbstractMojo { - @Parameter(defaultValue = "${project}") - private MavenProject project = null; - - //TODO: Combine with com.yahoo.container.plugin.mojo.GenerateOsgiManifestMojo.bundleSymbolicName - @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") - private String bundleSymbolicName = null; - - /* Sample output -- target/test-classes/bundle-plugin.bundle-classpath-mappings.json - { - "mainBundle": { - "bundleSymbolicName": "bundle-plugin-test", - "classPathElements": [ - "/Users/tonyv/Repos/vespa/bundle-plugin-test/target/classes", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/7-SNAPSHOT/jrt-7-SNAPSHOT.jar", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/annotations/7-SNAPSHOT/annotations-7-SNAPSHOT.jar" - ] - }, - "providedDependencies": [ - { - "bundleSymbolicName": "jrt", - "classPathElements": [ - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/7-SNAPSHOT/jrt-7-SNAPSHOT.jar" - ] - } - ] - } - */ - @Override - public void execute() throws MojoExecutionException { - Preconditions.checkNotNull(bundleSymbolicName); - - Artifacts.ArtifactSet artifacts = Artifacts.getArtifacts(project); - List<Artifact> embeddedArtifacts = artifacts.getJarArtifactsToInclude(); - List<Artifact> providedJarArtifacts = artifacts.getJarArtifactsProvided(); - - List<File> embeddedArtifactsFiles = embeddedArtifacts.stream().map(Artifact::getFile).collect(Collectors.toList()); - - List<String> classPathElements = Stream.concat(Stream.of(outputDirectory()), embeddedArtifactsFiles.stream()) - .map(File::getAbsolutePath).collect(Collectors.toList()); - - ProjectBundleClassPaths classPathMappings = new ProjectBundleClassPaths( - new BundleClasspathMapping(bundleSymbolicName, classPathElements), - providedJarArtifacts.stream().map(f -> createDependencyClasspathMapping(f)).filter(Optional::isPresent).map(Optional::get) - .collect(Collectors.toList())); - - try { - ProjectBundleClassPaths.save(testOutputPath().resolve(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME), classPathMappings); - } catch (IOException e) { - throw new MojoExecutionException("Error saving to file " + testOutputPath(), e); - } - } - - private File outputDirectory() { - return new File(project.getBuild().getOutputDirectory()); - } - - private Path testOutputPath() { - return Paths.get(project.getBuild().getTestOutputDirectory()); - } - - /* TODO: - * 1) add the dependencies of the artifact in the future(i.e. dependencies of dependencies) - * or - * 2) obtain bundles with embedded dependencies from the maven repository, - * and support loading classes from the nested jar files in those bundles. - */ - Optional<BundleClasspathMapping> createDependencyClasspathMapping(Artifact artifact) { - return bundleSymbolicNameForArtifact(artifact) - .map(name -> new BundleClasspathMapping(name, Arrays.asList(artifact.getFile().getAbsolutePath()))); - } - - private static Optional<String> bundleSymbolicNameForArtifact(Artifact artifact) { - if (artifact.getFile().getName().endsWith(".jar")) { - return AnalyzeBundle.bundleSymbolicName(artifact.getFile()); - } else { - // Not the best heuristic. The other alternatives are parsing the pom file or - // storing information in target/classes when building the provided bundles. - return Optional.of(artifact.getArtifactId()); - } - } -} diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ProjectBundleClassPaths.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ProjectBundleClassPaths.java deleted file mode 100644 index 42033f6ac73..00000000000 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/osgi/ProjectBundleClassPaths.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.plugin.osgi; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Objects; - -/** - * Represents the bundles in a maven project and the classpath elements - * corresponding to code that would end up in the bundle. - * - * @author Tony Vaagenes - * @author bjorncs - */ -public class ProjectBundleClassPaths { - public static final String CLASSPATH_MAPPINGS_FILENAME = "bundle-plugin.bundle-classpath-mappings.json"; - - private static final ObjectMapper mapper = new ObjectMapper(); - - @JsonProperty("mainBundle") public final BundleClasspathMapping mainBundle; - @JsonProperty("providedDependencies") public final List<BundleClasspathMapping> providedDependencies; - - @JsonCreator - public ProjectBundleClassPaths(@JsonProperty("mainBundle") BundleClasspathMapping mainBundle, - @JsonProperty("providedDependencies") List<BundleClasspathMapping> providedDependencies) { - this.mainBundle = mainBundle; - this.providedDependencies = providedDependencies; - } - - public static void save(Path path, ProjectBundleClassPaths mappings) throws IOException { - Files.createDirectories(path.getParent()); - mapper.writeValue(path.toFile(), mappings); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ProjectBundleClassPaths that = (ProjectBundleClassPaths) o; - return Objects.equals(mainBundle, that.mainBundle) && - Objects.equals(providedDependencies, that.providedDependencies); - } - - @Override - public int hashCode() { - return Objects.hash(mainBundle, providedDependencies); - } - - public static class BundleClasspathMapping { - @JsonProperty("bundleSymbolicName") public final String bundleSymbolicName; - @JsonProperty("classPathElements") public final List<String> classPathElements; - - @JsonCreator - public BundleClasspathMapping(@JsonProperty("bundleSymbolicName") String bundleSymbolicName, - @JsonProperty("classPathElements") List<String> classPathElements) { - this.bundleSymbolicName = bundleSymbolicName; - this.classPathElements = classPathElements; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BundleClasspathMapping that = (BundleClasspathMapping) o; - return Objects.equals(bundleSymbolicName, that.bundleSymbolicName) && - Objects.equals(classPathElements, that.classPathElements); - } - - @Override - public int hashCode() { - return Objects.hash(bundleSymbolicName, classPathElements); - } - } -} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 4b0e30e04da..944a989d79f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -89,7 +89,6 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean dryRunOnnxOnSetup() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.428") default int clusterControllerMaxHeapSizeInMb() { return 128; } - @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.422") default int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return 256; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"tokle"}) default boolean tenantIamRole() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index d66295d4c36..7d065fa93f4 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -411,6 +411,7 @@ sub http_upload { debug("exitcode=$exitcode\n"); debug("output=$output\n"); $configsource_url = shift(@configsources); + $configsource_url =~ s/\/$//; # Remove last / from configsource_url if ($configsource_url) { $configsource_url_used = $configsource_url; $retry = 1; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 56fd6a64305..1eb27a3224a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -107,7 +107,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index d228e6638fe..de2e4d54eec 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -49,6 +49,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.zookeeper.KeeperException; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -90,6 +91,7 @@ import static com.yahoo.vespa.curator.Curator.CompletionWaiter; public class SessionRepository { private static final Logger log = Logger.getLogger(SessionRepository.class.getName()); + private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); private static final long nonExistingActiveSessionId = 0; private final Object monitor = new Object(); @@ -203,6 +205,22 @@ public class SessionRepository { return List.copyOf(localSessionCache.values()); } + public Set<LocalSession> getLocalSessionsFromFileSystem() { + File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); + if (sessions == null) return Set.of(); + + Set<LocalSession> sessionIds = new HashSet<>(); + for (File session : sessions) { + long sessionId = Long.parseLong(session.getName()); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + LocalSession localSession = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient); + sessionIds.add(localSession); + } + return sessionIds; + } + public ConfigChangeActions prepareLocalSession(Session session, DeployLogger logger, PrepareParams params, Instant now) { applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( logger.log(Level.FINE, "Created application " + params.getApplicationId()); @@ -529,7 +547,7 @@ public class SessionRepository { log.log(Level.FINE, () -> "Purging old sessions for tenant '" + tenantName + "'"); Set<LocalSession> toDelete = new HashSet<>(); try { - for (LocalSession candidate : getLocalSessions()) { + for (LocalSession candidate : getLocalSessionsFromFileSystem()) { Instant createTime = candidate.getCreateTime(); log.log(Level.FINE, () -> "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 1b31d02d222..8f8005894e5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -465,12 +465,13 @@ public class ApplicationRepositoryTest { // Create a local session without any data in zookeeper (corner case seen in production occasionally) // and check that expiring local sessions still work int sessionId = 6; - Files.createDirectory(new TenantFileSystemDirs(serverdb, tenant1).getUserApplicationDir(sessionId).toPath()); + TenantName tenantName = tester.tenant().getName(); + Files.createDirectory(new TenantFileSystemDirs(serverdb, tenantName).getUserApplicationDir(sessionId).toPath()); LocalSession localSession2 = new LocalSession(tenant1, sessionId, FilesApplicationPackage.fromFile(testApp), new SessionZooKeeperClient(curator, - tenant1, + tenantName, sessionId, ConfigUtils.getCanonicalHostName())); sessionRepository.addLocalSession(localSession2); diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 08a468a3031..1d63fb2312a 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -5,7 +5,6 @@ import com.google.inject.AbstractModule; import com.google.inject.Inject; import com.google.inject.Injector; import com.yahoo.component.AbstractComponent; -import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.FileReference; @@ -13,8 +12,6 @@ import com.yahoo.container.di.ComponentDeconstructor; import com.yahoo.container.di.Container; import com.yahoo.container.di.componentgraph.core.ComponentGraph; import com.yahoo.container.di.config.SubscriberFactory; -import com.yahoo.container.di.osgi.BundleClasses; -import com.yahoo.container.di.osgi.OsgiUtil; import com.yahoo.container.logging.AccessLog; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.jdisc.application.OsgiFramework; @@ -25,7 +22,6 @@ import com.yahoo.osgi.OsgiImpl; import com.yahoo.osgi.OsgiWrapper; import com.yahoo.statistics.Statistics; import org.osgi.framework.Bundle; -import org.osgi.framework.wiring.BundleWiring; import java.util.ArrayList; import java.util.Collection; @@ -35,8 +31,6 @@ import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.collections.CollectionUtil.first; - /** * For internal use only. * @@ -94,27 +88,6 @@ public class HandlersConfigurerDi { platformBundleLoader = new PlatformBundleLoader(this); } - - // TODO Vespa 8: Remove, only used for Jersey - @Override - public BundleClasses getBundleClasses(ComponentSpecification bundleSpec, Set<String> packagesToScan) { - //Temporary hack: Using class name since ClassLoaderOsgiFramework is not available at compile time in this bundle. - if (osgiFramework.getClass().getName().equals("com.yahoo.application.container.impl.ClassLoaderOsgiFramework")) { - Bundle syntheticClassPathBundle = first(osgiFramework.bundles()); - ClassLoader classLoader = syntheticClassPathBundle.adapt(BundleWiring.class).getClassLoader(); - - return new BundleClasses( - syntheticClassPathBundle, - OsgiUtil.getClassEntriesForBundleUsingProjectClassPathMappings(classLoader, bundleSpec, packagesToScan)); - } else { - Bundle bundle = getBundle(bundleSpec); - if (bundle == null) - throw new RuntimeException("No bundle matching '" + bundleSpec + "'"); - - return new BundleClasses(bundle, OsgiUtil.getClassEntriesInBundleClassPath(bundle, packagesToScan)); - } - } - @Override public void installPlatformBundles(Collection<String> bundlePaths) { // Don't install physical bundles for test frameworks, where all platform bundles are on the classpath. diff --git a/container-core/src/main/java/com/yahoo/container/di/Container.java b/container-core/src/main/java/com/yahoo/container/di/Container.java index a91aa6eb588..2bd8b2a90ec 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Container.java +++ b/container-core/src/main/java/com/yahoo/container/di/Container.java @@ -12,11 +12,9 @@ import com.yahoo.container.di.ConfigRetriever.ComponentsConfigs; import com.yahoo.container.di.ConfigRetriever.ConfigSnapshot; import com.yahoo.container.di.componentgraph.core.ComponentGraph; import com.yahoo.container.di.componentgraph.core.ComponentNode; -import com.yahoo.container.di.componentgraph.core.JerseyNode; import com.yahoo.container.di.componentgraph.core.Node; import com.yahoo.container.di.config.ApplicationBundlesConfig; import com.yahoo.container.di.config.PlatformBundlesConfig; -import com.yahoo.container.di.config.RestApiContext; import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.vespa.config.ConfigKey; import org.osgi.framework.Bundle; @@ -200,14 +198,7 @@ public class Container { for (ComponentsConfig.Components config : componentsConfig.components()) { BundleInstantiationSpecification specification = bundleInstantiationSpecification(config); Class<?> componentClass = osgi.resolveClass(specification); - Node componentNode; - - if (RestApiContext.class.isAssignableFrom(componentClass)) { - Class<? extends RestApiContext> nodeClass = componentClass.asSubclass(RestApiContext.class); - componentNode = new JerseyNode(specification.id, config.configId(), nodeClass, osgi); - } else { - componentNode = new ComponentNode(specification.id, config.configId(), componentClass, null); - } + Node componentNode = new ComponentNode(specification.id, config.configId(), componentClass, null); graph.add(componentNode); } } diff --git a/container-core/src/main/java/com/yahoo/container/di/Osgi.java b/container-core/src/main/java/com/yahoo/container/di/Osgi.java index 940986e2f38..2ba93171081 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Osgi.java +++ b/container-core/src/main/java/com/yahoo/container/di/Osgi.java @@ -5,11 +5,9 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.config.FileReference; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.bundle.MockBundle; -import com.yahoo.container.di.osgi.BundleClasses; import org.osgi.framework.Bundle; import java.util.Collection; -import java.util.Collections; import java.util.Set; import java.util.stream.Collectors; @@ -21,9 +19,6 @@ import static java.util.Collections.emptySet; * @author ollivir */ public interface Osgi { - default BundleClasses getBundleClasses(ComponentSpecification bundle, Set<String> packagesToScan) { - return new BundleClasses(new MockBundle(), Collections.emptySet()); - } default void installPlatformBundles(Collection<String> bundlePaths) { System.out.println("installPlatformBundles " + bundlePaths); diff --git a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/JerseyNode.java b/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/JerseyNode.java deleted file mode 100644 index 0f8aa678934..00000000000 --- a/container-core/src/main/java/com/yahoo/container/di/componentgraph/core/JerseyNode.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.di.componentgraph.core; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.container.di.Osgi; -import com.yahoo.container.di.config.JerseyBundlesConfig; -import com.yahoo.container.di.config.RestApiContext; -import com.yahoo.container.di.config.RestApiContext.BundleInfo; -import com.yahoo.container.di.osgi.BundleClasses; -import org.osgi.framework.Bundle; -import org.osgi.framework.wiring.BundleWiring; - -import java.net.URL; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; - -/** - * Represents an instance of RestApiContext - * - * @author gjoranv - * @author Tony Vaagenes - * @author ollivir - */ -public class JerseyNode extends ComponentNode { - private static final String WEB_INF_URL = "WebInfUrl"; - - private final Osgi osgi; - - public JerseyNode(ComponentId componentId, String configId, Class<?> clazz, Osgi osgi) { - super(componentId, configId, clazz, null); - this.osgi = osgi; - } - - @Override - protected RestApiContext newInstance() { - Object instance = super.newInstance(); - RestApiContext restApiContext = (RestApiContext) instance; - - List<JerseyBundlesConfig.Bundles> bundles = restApiContext.bundlesConfig.bundles(); - for (JerseyBundlesConfig.Bundles bundleConfig : bundles) { - BundleClasses bundleClasses = osgi.getBundleClasses(ComponentSpecification.fromString(bundleConfig.spec()), - new HashSet<>(bundleConfig.packages())); - - restApiContext.addBundle(createBundleInfo(bundleClasses.bundle(), bundleClasses.classEntries())); - } - - componentsToInject.forEach(component -> restApiContext.addInjectableComponent(component.instanceKey(), component.componentId(), - component.component())); - - return restApiContext; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - @Override - public boolean equals(Object other) { - return super.equals(other) - && (other instanceof JerseyNode && this.componentsToInject.equals(((JerseyNode) other).componentsToInject)); - } - - public static BundleInfo createBundleInfo(Bundle bundle, Collection<String> classEntries) { - BundleInfo bundleInfo = new BundleInfo(bundle.getSymbolicName(), bundle.getVersion(), bundle.getLocation(), webInfUrl(bundle), - bundle.adapt(BundleWiring.class).getClassLoader()); - - bundleInfo.setClassEntries(classEntries); - return bundleInfo; - } - - public static Bundle getBundle(Osgi osgi, String bundleSpec) { - Bundle bundle = osgi.getBundle(ComponentSpecification.fromString(bundleSpec)); - if (bundle == null) { - throw new IllegalArgumentException("Bundle not found: " + bundleSpec); - } - return bundle; - } - - private static URL webInfUrl(Bundle bundle) { - String webInfUrlHeader = bundle.getHeaders().get(WEB_INF_URL); - - if (webInfUrlHeader == null) { - return null; - } else { - return bundle.getEntry(webInfUrlHeader); - } - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/di/config/RestApiContext.java b/container-core/src/main/java/com/yahoo/container/di/config/RestApiContext.java deleted file mode 100644 index bfb9a8f9160..00000000000 --- a/container-core/src/main/java/com/yahoo/container/di/config/RestApiContext.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.di.config; - -import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; -import com.google.inject.Key; -import com.yahoo.component.ComponentId; -import org.osgi.framework.Version; - -import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -/** - * Only for internal JDisc use. - * - * @author gjoranv - */ -public class RestApiContext { - - private final List<BundleInfo> bundles = new ArrayList<>(); - private final List<Injectable> injectableComponents = new ArrayList<>(); - - public final JerseyBundlesConfig bundlesConfig; - public final JerseyInjectionConfig injectionConfig; - - @Inject - public RestApiContext(JerseyBundlesConfig bundlesConfig, JerseyInjectionConfig injectionConfig) { - this.bundlesConfig = bundlesConfig; - this.injectionConfig = injectionConfig; - } - - public List<BundleInfo> getBundles() { - return Collections.unmodifiableList(bundles); - } - - public void addBundle(BundleInfo bundle) { - bundles.add(bundle); - } - - public List<Injectable> getInjectableComponents() { - return Collections.unmodifiableList(injectableComponents); - } - - public void addInjectableComponent(Key<?> key, ComponentId id, Object component) { - injectableComponents.add(new Injectable(key, id, component)); - } - - public static class Injectable { - public final Key<?> key; - public final ComponentId id; - public final Object instance; - - public Injectable(Key<?> key, ComponentId id, Object instance) { - this.key = key; - this.id = id; - this.instance = instance; - } - @Override - public String toString() { - return id.toString(); - } - } - - public static class BundleInfo { - public final String symbolicName; - public final Version version; - public final String fileLocation; - public final URL webInfUrl; - public final ClassLoader classLoader; - - private Set<String> classEntries; - - public BundleInfo(String symbolicName, Version version, String fileLocation, URL webInfUrl, ClassLoader classLoader) { - this.symbolicName = symbolicName; - this.version = version; - this.fileLocation = fileLocation; - this.webInfUrl = webInfUrl; - this.classLoader = classLoader; - } - - @Override - public String toString() { - return symbolicName + ":" + version; - } - - public void setClassEntries(Collection<String> entries) { - this.classEntries = ImmutableSet.copyOf(entries); - } - - public Set<String> getClassEntries() { - return classEntries; - } - } -} diff --git a/container-core/src/main/java/com/yahoo/container/di/osgi/BundleClasses.java b/container-core/src/main/java/com/yahoo/container/di/osgi/BundleClasses.java deleted file mode 100644 index bca3ed73d0b..00000000000 --- a/container-core/src/main/java/com/yahoo/container/di/osgi/BundleClasses.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.di.osgi; - -import org.osgi.framework.Bundle; - -import java.util.Collection; - -/** - * @author ollivir - */ -public class BundleClasses { - private final Bundle bundle; - private final Collection<String> classEntries; - - public BundleClasses(Bundle bundle, Collection<String> classEntries) { - this.bundle = bundle; - this.classEntries = classEntries; - } - - public Bundle bundle() { - return bundle; - } - - public Collection<String> classEntries() { - return classEntries; - } -} diff --git a/container-core/src/main/java/com/yahoo/container/di/osgi/OsgiUtil.java b/container-core/src/main/java/com/yahoo/container/di/osgi/OsgiUtil.java deleted file mode 100644 index e1854155e5b..00000000000 --- a/container-core/src/main/java/com/yahoo/container/di/osgi/OsgiUtil.java +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.di.osgi; - -import com.yahoo.component.ComponentSpecification; -import com.yahoo.osgi.maven.ProjectBundleClassPaths; -import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping; -import org.osgi.framework.Bundle; -import org.osgi.framework.wiring.BundleWiring; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.function.Predicate; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.google.common.io.Files.fileTreeTraverser; - -/** - * Tested by com.yahoo.application.container.jersey.JerseyTest - * - * @author Tony Vaagenes - * @author ollivir - */ -public class OsgiUtil { - private static final Logger log = Logger.getLogger(OsgiUtil.class.getName()); - private static final String CLASS_FILE_TYPE_SUFFIX = ".class"; - - public static Collection<String> getClassEntriesInBundleClassPath(Bundle bundle, Set<String> packagesToScan) { - BundleWiring bundleWiring = bundle.adapt(BundleWiring.class); - - if (packagesToScan.isEmpty()) { - return bundleWiring.listResources("/", "*" + CLASS_FILE_TYPE_SUFFIX, - BundleWiring.LISTRESOURCES_LOCAL | BundleWiring.LISTRESOURCES_RECURSE); - } else { - List<String> ret = new ArrayList<>(); - for (String pkg : packagesToScan) { - ret.addAll(bundleWiring.listResources(packageToPath(pkg), "*" + CLASS_FILE_TYPE_SUFFIX, BundleWiring.LISTRESOURCES_LOCAL)); - } - return ret; - } - } - - public static Collection<String> getClassEntriesForBundleUsingProjectClassPathMappings(ClassLoader classLoader, - ComponentSpecification bundleSpec, Set<String> packagesToScan) { - return classEntriesFrom(bundleClassPathMapping(bundleSpec, classLoader).classPathElements, packagesToScan); - } - - private static BundleClasspathMapping bundleClassPathMapping(ComponentSpecification bundleSpec, ClassLoader classLoader) { - ProjectBundleClassPaths projectBundleClassPaths = loadProjectBundleClassPaths(classLoader); - - if (projectBundleClassPaths.mainBundle.bundleSymbolicName.equals(bundleSpec.getName())) { - return projectBundleClassPaths.mainBundle; - } else { - log.log(Level.WARNING, - "Dependencies of the bundle " + bundleSpec + " will not be scanned. Please file a feature request if you need this"); - return matchingBundleClassPathMapping(bundleSpec, projectBundleClassPaths.providedDependencies); - } - } - - public static BundleClasspathMapping matchingBundleClassPathMapping(ComponentSpecification bundleSpec, - Collection<BundleClasspathMapping> providedBundlesClassPathMappings) { - for (BundleClasspathMapping mapping : providedBundlesClassPathMappings) { - if (mapping.bundleSymbolicName.equals(bundleSpec.getName())) { - return mapping; - } - } - throw new RuntimeException("No such bundle: " + bundleSpec); - } - - private static ProjectBundleClassPaths loadProjectBundleClassPaths(ClassLoader classLoader) { - URL classPathMappingsFileLocation = classLoader.getResource(ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME); - if (classPathMappingsFileLocation == null) { - throw new RuntimeException("Couldn't find " + ProjectBundleClassPaths.CLASSPATH_MAPPINGS_FILENAME + " in the class path."); - } - - try { - return ProjectBundleClassPaths.load(Paths.get(classPathMappingsFileLocation.toURI())); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - - private static Collection<String> classEntriesFrom(List<String> classPathEntries, Set<String> packagesToScan) { - Set<String> packagePathsToScan = packagesToScan.stream().map(OsgiUtil::packageToPath).collect(Collectors.toSet()); - List<String> ret = new ArrayList<>(); - - for (String entry : classPathEntries) { - Path path = Paths.get(entry); - if (Files.isDirectory(path)) { - ret.addAll(classEntriesInPath(path, packagePathsToScan)); - } else if (Files.isRegularFile(path) && path.getFileName().toString().endsWith(".jar")) { - ret.addAll(classEntriesInJar(path, packagePathsToScan)); - } else { - throw new RuntimeException("Unsupported path " + path + " in the class path"); - } - } - return ret; - } - - private static String relativePathToClass(Path rootPath, Path pathToClass) { - Path relativePath = rootPath.relativize(pathToClass); - return relativePath.toString(); - } - - private static Collection<String> classEntriesInPath(Path rootPath, Collection<String> packagePathsToScan) { - Iterable<File> fileIterator; - if (packagePathsToScan.isEmpty()) { - fileIterator = fileTreeTraverser().preOrderTraversal(rootPath.toFile()); - } else { - List<File> files = new ArrayList<>(); - for (String packagePath : packagePathsToScan) { - for (File file : fileTreeTraverser().children(rootPath.resolve(packagePath).toFile())) { - files.add(file); - } - } - fileIterator = files; - } - - List<String> ret = new ArrayList<>(); - for (File file : fileIterator) { - if (file.isFile() && file.getName().endsWith(CLASS_FILE_TYPE_SUFFIX)) { - ret.add(relativePathToClass(rootPath, file.toPath())); - } - } - return ret; - } - - private static String packagePath(String name) { - int index = name.lastIndexOf('/'); - if (index < 0) { - return name; - } else { - return name.substring(0, index); - } - } - - private static Collection<String> classEntriesInJar(Path jarPath, Set<String> packagePathsToScan) { - Predicate<String> acceptedPackage; - if (packagePathsToScan.isEmpty()) { - acceptedPackage = ign -> true; - } else { - acceptedPackage = name -> packagePathsToScan.contains(packagePath(name)); - } - - try (JarFile jarFile = new JarFile(jarPath.toFile())) { - return jarFile.stream().map(JarEntry::getName).filter(name -> name.endsWith(CLASS_FILE_TYPE_SUFFIX)).filter(acceptedPackage) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static String packageToPath(String packageName) { - return packageName.replace('.', '/'); - } -} diff --git a/container-core/src/main/java/com/yahoo/container/di/osgi/package-info.java b/container-core/src/main/java/com/yahoo/container/di/osgi/package-info.java deleted file mode 100644 index 9685cf571bd..00000000000 --- a/container-core/src/main/java/com/yahoo/container/di/osgi/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author Tony Vaagenes - */ -@ExportPackage -package com.yahoo.container.di.osgi; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java index 6f5bf298de3..2ef14679553 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApi.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -2,8 +2,10 @@ package com.yahoo.restapi; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.RequestHandlerSpec; import java.io.InputStream; import java.util.List; @@ -20,38 +22,78 @@ public interface RestApi { static Builder builder() { return new RestApiImpl.BuilderImpl(); } static RouteBuilder route(String pathPattern) { return new RestApiImpl.RouteBuilderImpl(pathPattern); } + static HandlerConfigBuilder handlerConfig() { return new RestApiImpl.HandlerConfigBuilderImpl(); } HttpResponse handleRequest(HttpRequest request); ObjectMapper jacksonJsonMapper(); + /** @see com.yahoo.container.jdisc.HttpRequestHandler#requestHandlerSpec() */ + RequestHandlerSpec requestHandlerSpec(); + interface Builder { Builder setObjectMapper(ObjectMapper mapper); Builder setDefaultRoute(RouteBuilder route); Builder addRoute(RouteBuilder route); Builder addFilter(Filter filter); + /** see {@link RestApiMappers#DEFAULT_EXCEPTION_MAPPERS} for default mappers */ <EXCEPTION extends RuntimeException> Builder addExceptionMapper(Class<EXCEPTION> type, ExceptionMapper<EXCEPTION> mapper); + /** see {@link RestApiMappers#DEFAULT_RESPONSE_MAPPERS} for default mappers */ <RESPONSE_ENTITY> Builder addResponseMapper(Class<RESPONSE_ENTITY> type, ResponseMapper<RESPONSE_ENTITY> mapper); + /** see {@link RestApiMappers#DEFAULT_REQUEST_MAPPERS} for default mappers */ <REQUEST_ENTITY> Builder addRequestMapper(Class<REQUEST_ENTITY> type, RequestMapper<REQUEST_ENTITY> mapper); <RESPONSE_ENTITY> Builder registerJacksonResponseEntity(Class<RESPONSE_ENTITY> type); <REQUEST_ENTITY> Builder registerJacksonRequestEntity(Class<REQUEST_ENTITY> type); + /** Disables mappers listed in {@link RestApiMappers#DEFAULT_EXCEPTION_MAPPERS} */ Builder disableDefaultExceptionMappers(); + /** Disables mappers listed in {@link RestApiMappers#DEFAULT_RESPONSE_MAPPERS} */ Builder disableDefaultResponseMappers(); + Builder disableDefaultAclMapping(); RestApi build(); } interface RouteBuilder { RouteBuilder name(String name); + RouteBuilder addFilter(Filter filter); + + // GET RouteBuilder get(Handler<?> handler); + RouteBuilder get(Handler<?> handler, HandlerConfigBuilder config); + + // POST RouteBuilder post(Handler<?> handler); - <REQUEST_ENTITY> RouteBuilder post(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + <REQUEST_ENTITY> RouteBuilder post( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + RouteBuilder post(Handler<?> handler, HandlerConfigBuilder config); + <REQUEST_ENTITY> RouteBuilder post( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config); + + // PUT RouteBuilder put(Handler<?> handler); - <REQUEST_ENTITY> RouteBuilder put(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + <REQUEST_ENTITY> RouteBuilder put( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + RouteBuilder put(Handler<?> handler, HandlerConfigBuilder config); + <REQUEST_ENTITY> RouteBuilder put( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config); + + // DELETE RouteBuilder delete(Handler<?> handler); + RouteBuilder delete(Handler<?> handler, HandlerConfigBuilder config); + + // PATCH RouteBuilder patch(Handler<?> handler); - <REQUEST_ENTITY> RouteBuilder patch(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + <REQUEST_ENTITY> RouteBuilder patch( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + RouteBuilder patch(Handler<?> handler, HandlerConfigBuilder config); + <REQUEST_ENTITY> RouteBuilder patch( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config); + + // Default RouteBuilder defaultHandler(Handler<?> handler); - <REQUEST_ENTITY> RouteBuilder defaultHandler(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); - RouteBuilder addFilter(Filter filter); + RouteBuilder defaultHandler(Handler<?> handler, HandlerConfigBuilder config); + <REQUEST_ENTITY> RouteBuilder defaultHandler( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + <REQUEST_ENTITY> RouteBuilder defaultHandler( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config); } @FunctionalInterface interface Handler<RESPONSE_ENTITY> { @@ -70,6 +112,12 @@ public interface RestApi { @FunctionalInterface interface Filter { HttpResponse filterRequest(FilterContext context); } + interface HandlerConfigBuilder { + HandlerConfigBuilder withReadAclAction(); + HandlerConfigBuilder withWriteAclAction(); + HandlerConfigBuilder withCustomAclAction(AclMapping.Action action); + } + interface RequestContext { HttpRequest request(); PathParameters pathParameters(); @@ -80,6 +128,7 @@ public interface RestApi { RequestContent requestContentOrThrow(); ObjectMapper jacksonJsonMapper(); UriBuilder uriBuilder(); + AclMapping.Action aclAction(); interface Parameters { Optional<String> getString(String name); diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index d63add5ed1d..646177e60db 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -1,20 +1,20 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.RequestView; import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.yolean.Exceptions; +import com.yahoo.restapi.RestApiMappers.ExceptionMapperHolder; +import com.yahoo.restapi.RestApiMappers.RequestMapperHolder; +import com.yahoo.restapi.RestApiMappers.ResponseMapperHolder; -import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.ListIterator; @@ -23,8 +23,6 @@ import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; -import static java.nio.charset.StandardCharsets.UTF_8; - /** * @author bjorncs */ @@ -39,6 +37,7 @@ class RestApiImpl implements RestApi { private final List<RequestMapperHolder<?>> requestMappers; private final List<Filter> filters; private final ObjectMapper jacksonJsonMapper; + private final boolean disableDefaultAclMapping; private RestApiImpl(RestApi.Builder builder) { BuilderImpl builderImpl = (BuilderImpl) builder; @@ -48,18 +47,19 @@ class RestApiImpl implements RestApi { this.exceptionMappers = combineWithDefaultExceptionMappers( builderImpl.exceptionMappers, Boolean.TRUE.equals(builderImpl.disableDefaultExceptionMappers)); this.responseMappers = combineWithDefaultResponseMappers( - builderImpl.responseMappers, jacksonJsonMapper, Boolean.TRUE.equals(builderImpl.disableDefaultResponseMappers)); - this.requestMappers = combineWithDefaultRequestMappers( - builderImpl.requestMappers, jacksonJsonMapper); + builderImpl.responseMappers, Boolean.TRUE.equals(builderImpl.disableDefaultResponseMappers)); + this.requestMappers = combineWithDefaultRequestMappers(builderImpl.requestMappers); this.filters = List.copyOf(builderImpl.filters); this.jacksonJsonMapper = jacksonJsonMapper; + this.disableDefaultAclMapping = Boolean.TRUE.equals(builderImpl.disableDefaultAclMapping); } @Override public HttpResponse handleRequest(HttpRequest request) { Path pathMatcher = new Path(request.getUri()); Route resolvedRoute = resolveRoute(pathMatcher); - RequestContextImpl requestContext = new RequestContextImpl(request, pathMatcher, jacksonJsonMapper); + AclMapping.Action aclAction = getAclMapping(request.getMethod(), request.getUri()); + RequestContextImpl requestContext = new RequestContextImpl(request, pathMatcher, aclAction, jacksonJsonMapper); FilterContextImpl filterContext = createFilterContextRecursive( resolvedRoute, requestContext, filters, @@ -73,8 +73,33 @@ class RestApiImpl implements RestApi { @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } + @Override + public RequestHandlerSpec requestHandlerSpec() { + return RequestHandlerSpec.builder() + .withAclMapping(requestView -> getAclMapping(requestView.method(), requestView.uri())) + .build(); + } + + private AclMapping.Action getAclMapping(Method method, URI uri) { + Path pathMatcher = new Path(uri); + Route route = resolveRoute(pathMatcher); + HandlerHolder<?> handler = resolveHandler(method, route); + AclMapping.Action aclAction = handler.config.aclAction; + if (aclAction != null) return aclAction; + if (!disableDefaultAclMapping) { + // Fallback to default request handler spec which is used by the default implementation of + // HttpRequestHandler.requestHandlerSpec(). + return RequestHandlerSpec.DEFAULT_INSTANCE.aclMapping().get( + new RequestView() { + @Override public Method method() { return method; } + @Override public URI uri() { return uri; } + }); + } + throw new IllegalStateException(String.format("No ACL mapping configured for '%s' to '%s'", method, route.name)); + } + private HttpResponse dispatchToRoute(Route route, RequestContextImpl context) { - HandlerHolder<?> resolvedHandler = resolveHandler(context, route); + HandlerHolder<?> resolvedHandler = resolveHandler(context.request.getMethod(), route); RequestMapperHolder<?> resolvedRequestMapper = resolveRequestMapper(resolvedHandler); Object requestEntity; try { @@ -97,8 +122,8 @@ class RestApiImpl implements RestApi { } } - private HandlerHolder<?> resolveHandler(RequestContextImpl context, Route route) { - HandlerHolder<?> resolvedHandler = route.handlerPerMethod.get(context.request().getMethod()); + private HandlerHolder<?> resolveHandler(Method method, Route route) { + HandlerHolder<?> resolvedHandler = route.handlerPerMethod.get(method); return resolvedHandler == null ? route.defaultHandler : resolvedHandler; } @@ -154,7 +179,7 @@ class RestApiImpl implements RestApi { List<ExceptionMapperHolder<?>> configuredExceptionMappers, boolean disableDefaultMappers) { List<ExceptionMapperHolder<?>> exceptionMappers = new ArrayList<>(configuredExceptionMappers); if (!disableDefaultMappers){ - exceptionMappers.add(new ExceptionMapperHolder<>(RestApiException.class, (context, exception) -> exception.response())); + exceptionMappers.addAll(RestApiMappers.DEFAULT_EXCEPTION_MAPPERS); } // Topologically sort children before superclasses, so most the specific match is found by iterating through mappers in order. exceptionMappers.sort((a, b) -> (a.type.isAssignableFrom(b.type) ? 1 : 0) + (b.type.isAssignableFrom(a.type) ? -1 : 0)); @@ -162,71 +187,21 @@ class RestApiImpl implements RestApi { } private static List<ResponseMapperHolder<?>> combineWithDefaultResponseMappers( - List<ResponseMapperHolder<?>> configuredResponseMappers, ObjectMapper jacksonJsonMapper, boolean disableDefaultMappers) { + List<ResponseMapperHolder<?>> configuredResponseMappers, boolean disableDefaultMappers) { List<ResponseMapperHolder<?>> responseMappers = new ArrayList<>(configuredResponseMappers); if (!disableDefaultMappers) { - responseMappers.add(new ResponseMapperHolder<>(HttpResponse.class, (context, entity) -> entity)); - responseMappers.add(new ResponseMapperHolder<>(String.class, (context, entity) -> new MessageResponse(entity))); - responseMappers.add(new ResponseMapperHolder<>(Slime.class, (context, entity) -> new SlimeJsonResponse(entity))); - responseMappers.add(new ResponseMapperHolder<>(JsonNode.class, (context, entity) -> new JacksonJsonResponse<>(200, entity, jacksonJsonMapper, true))); + responseMappers.addAll(RestApiMappers.DEFAULT_RESPONSE_MAPPERS); } return responseMappers; } private static List<RequestMapperHolder<?>> combineWithDefaultRequestMappers( - List<RequestMapperHolder<?>> configuredRequestMappers, ObjectMapper jacksonJsonMapper) { + List<RequestMapperHolder<?>> configuredRequestMappers) { List<RequestMapperHolder<?>> requestMappers = new ArrayList<>(configuredRequestMappers); - requestMappers.add(new RequestMapperHolder<>(Slime.class, RestApiImpl::toSlime)); - requestMappers.add(new RequestMapperHolder<>(JsonNode.class, ctx -> toJsonNode(ctx, jacksonJsonMapper))); - requestMappers.add(new RequestMapperHolder<>(String.class, RestApiImpl::toString)); - requestMappers.add(new RequestMapperHolder<>(byte[].class, RestApiImpl::toByteArray)); - requestMappers.add(new RequestMapperHolder<>(InputStream.class, RestApiImpl::toInputStream)); - requestMappers.add(new RequestMapperHolder<>(Void.class, ctx -> Optional.empty())); + requestMappers.addAll(RestApiMappers.DEFAULT_REQUEST_MAPPERS); return requestMappers; } - private static Optional<InputStream> toInputStream(RequestContext context) { - return context.requestContent().map(RequestContext.RequestContent::content); - } - - private static Optional<byte[]> toByteArray(RequestContext context) { - InputStream in = toInputStream(context).orElse(null); - if (in == null) return Optional.empty(); - return convertIoException(() -> Optional.of(in.readAllBytes())); - } - - private static Optional<String> toString(RequestContext context) { - try { - return toByteArray(context).map(bytes -> new String(bytes, UTF_8)); - } catch (RuntimeException e) { - throw new RestApiException.BadRequest("Failed parse request content as UTF-8: " + Exceptions.toMessageString(e), e); - } - } - - private static Optional<JsonNode> toJsonNode(RequestContext context, ObjectMapper jacksonJsonMapper) { - if (log.isLoggable(Level.FINE)) { - return toString(context).map(string -> { - log.fine(() -> "Request content: " + string); - return convertIoException("Failed to parse JSON", () -> jacksonJsonMapper.readTree(string)); - }); - } else { - return toInputStream(context) - .map(in -> convertIoException("Invalid JSON", () -> jacksonJsonMapper.readTree(in))); - } - } - - private static Optional<Slime> toSlime(RequestContext context) { - try { - return toString(context).map(string -> { - log.fine(() -> "Request content: " + string); - return SlimeUtils.jsonToSlimeOrThrow(string); - }); - } catch (com.yahoo.slime.JsonParseException e) { - log.log(Level.FINE, e.getMessage(), e); - throw new RestApiException.BadRequest("Invalid JSON: " + Exceptions.toMessageString(e), e); - } - } - static class BuilderImpl implements RestApi.Builder { private final List<Route> routes = new ArrayList<>(); private final List<ExceptionMapperHolder<?>> exceptionMappers = new ArrayList<>(); @@ -237,6 +212,7 @@ class RestApiImpl implements RestApi { private ObjectMapper jacksonJsonMapper; private Boolean disableDefaultExceptionMappers; private Boolean disableDefaultResponseMappers; + private Boolean disableDefaultAclMapping; @Override public RestApi.Builder setObjectMapper(ObjectMapper mapper) { this.jacksonJsonMapper = mapper; return this; } @Override public RestApi.Builder setDefaultRoute(RestApi.RouteBuilder route) { this.defaultRoute = ((RouteBuilderImpl)route).build(); return this; } @@ -256,20 +232,21 @@ class RestApiImpl implements RestApi { } @Override public <ENTITY> Builder registerJacksonResponseEntity(Class<ENTITY> type) { - addResponseMapper(type, new JacksonResponseMapper<>()); return this; + addResponseMapper(type, new RestApiMappers.JacksonResponseMapper<>()); return this; } @Override public <ENTITY> Builder registerJacksonRequestEntity(Class<ENTITY> type) { - addRequestMapper(type, new JacksonRequestMapper<>(type)); return this; + addRequestMapper(type, new RestApiMappers.JacksonRequestMapper<>(type)); return this; } @Override public Builder disableDefaultExceptionMappers() { this.disableDefaultExceptionMappers = true; return this; } @Override public Builder disableDefaultResponseMappers() { this.disableDefaultResponseMappers = true; return this; } + @Override public Builder disableDefaultAclMapping() { this.disableDefaultAclMapping = true; return this; } @Override public RestApi build() { return new RestApiImpl(this); } } - public static class RouteBuilderImpl implements RestApi.RouteBuilder { + static class RouteBuilderImpl implements RestApi.RouteBuilder { private final String pathPattern; private String name; private final Map<Method, HandlerHolder<?>> handlerPerMethod = new HashMap<>(); @@ -279,50 +256,118 @@ class RestApiImpl implements RestApi { RouteBuilderImpl(String pathPattern) { this.pathPattern = pathPattern; } @Override public RestApi.RouteBuilder name(String name) { this.name = name; return this; } - @Override public RestApi.RouteBuilder get(Handler<?> handler) { - return addHandler(Method.GET, handler); + @Override public RestApi.RouteBuilder addFilter(RestApi.Filter filter) { filters.add(filter); return this; } + + // GET + @Override public RouteBuilder get(Handler<?> handler) { return get(handler, null); } + @Override public RouteBuilder get(Handler<?> handler, HandlerConfigBuilder config) { + return addHandler(Method.GET, handler, config); } - @Override public RestApi.RouteBuilder post(Handler<?> handler) { - return addHandler(Method.POST, handler); + + // POST + @Override public RouteBuilder post(Handler<?> handler) { return post(handler, null); } + @Override public <REQUEST_ENTITY> RouteBuilder post( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler) { + return post(type, handler, null); } - @Override public <ENTITY> RouteBuilder post(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { - return addHandler(Method.POST, type, handler); + @Override public RouteBuilder post(Handler<?> handler, HandlerConfigBuilder config) { + return addHandler(Method.POST, handler, config); } - @Override public RestApi.RouteBuilder put(Handler<?> handler) { - return addHandler(Method.PUT, handler); + @Override public <REQUEST_ENTITY> RouteBuilder post( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config) { + return addHandler(Method.POST, type, handler, config); } - @Override public <ENTITY> RouteBuilder put(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { - return addHandler(Method.PUT, type, handler); + + // PUT + @Override public RouteBuilder put(Handler<?> handler) { return put(handler, null); } + @Override public <REQUEST_ENTITY> RouteBuilder put( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler) { + return put(type, handler, null); } - @Override public RestApi.RouteBuilder delete(Handler<?> handler) { - return addHandler(Method.DELETE, handler); + @Override public RouteBuilder put(Handler<?> handler, HandlerConfigBuilder config) { + return addHandler(Method.PUT, handler, null); } - @Override public RestApi.RouteBuilder patch(Handler<?> handler) { - return addHandler(Method.PATCH, handler); + @Override public <REQUEST_ENTITY> RouteBuilder put( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config) { + return addHandler(Method.PUT, type, handler, config); } - @Override public <ENTITY> RouteBuilder patch(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { - return addHandler(Method.PATCH, type, handler); + + // DELETE + @Override public RouteBuilder delete(Handler<?> handler) { return delete(handler, null); } + @Override public RouteBuilder delete(Handler<?> handler, HandlerConfigBuilder config) { + return addHandler(Method.DELETE, handler, config); } - @Override public RestApi.RouteBuilder defaultHandler(Handler<?> handler) { - defaultHandler = HandlerHolder.of(handler); return this; + + // PATCH + @Override public RouteBuilder patch(Handler<?> handler) { return patch(handler, null); } + @Override public <REQUEST_ENTITY> RouteBuilder patch( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler) { + return patch(type, handler, null); } - @Override public <ENTITY> RouteBuilder defaultHandler(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { - defaultHandler = HandlerHolder.of(type, handler); return this; + @Override public RouteBuilder patch(Handler<?> handler, HandlerConfigBuilder config) { + return addHandler(Method.PATCH, handler, config); + } + @Override public <REQUEST_ENTITY> RouteBuilder patch( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config) { + return addHandler(Method.PATCH, type, handler, config); } - @Override public RestApi.RouteBuilder addFilter(RestApi.Filter filter) { filters.add(filter); return this; } - private RestApi.RouteBuilder addHandler(Method method, Handler<?> handler) { - handlerPerMethod.put(method, HandlerHolder.of(handler)); return this; + // Default + @Override public RouteBuilder defaultHandler(Handler<?> handler) { + return defaultHandler(handler, null); + } + @Override public RouteBuilder defaultHandler(Handler<?> handler, HandlerConfigBuilder config) { + defaultHandler = HandlerHolder.of(handler, build(config)); return this; + } + @Override public <REQUEST_ENTITY> RouteBuilder defaultHandler( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler) { + return defaultHandler(type, handler, null); + } + @Override + public <REQUEST_ENTITY> RouteBuilder defaultHandler( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, HandlerConfigBuilder config) { + defaultHandler = HandlerHolder.of(type, handler, build(config)); return this; + } + + private RestApi.RouteBuilder addHandler(Method method, Handler<?> handler, HandlerConfigBuilder config) { + handlerPerMethod.put(method, HandlerHolder.of(handler, build(config))); return this; } private <ENTITY> RestApi.RouteBuilder addHandler( - Method method, Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { - handlerPerMethod.put(method, HandlerHolder.of(type, handler)); return this; + Method method, Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler, HandlerConfigBuilder config) { + handlerPerMethod.put(method, HandlerHolder.of(type, handler, build(config))); return this; + } + + private static HandlerConfig build(HandlerConfigBuilder builder) { + if (builder == null) return HandlerConfig.empty(); + return ((HandlerConfigBuilderImpl)builder).build(); } private Route build() { return new Route(this); } } + static class HandlerConfigBuilderImpl implements HandlerConfigBuilder { + private AclMapping.Action aclAction; + + @Override public HandlerConfigBuilder withReadAclAction() { return withCustomAclAction(AclMapping.Action.READ); } + @Override public HandlerConfigBuilder withWriteAclAction() { return withCustomAclAction(AclMapping.Action.WRITE); } + @Override public HandlerConfigBuilder withCustomAclAction(AclMapping.Action action) { + this.aclAction = action; return this; + } + + HandlerConfig build() { return new HandlerConfig(this); } + } + + private static class HandlerConfig { + final AclMapping.Action aclAction; + + HandlerConfig(HandlerConfigBuilderImpl builder) { + this.aclAction = builder.aclAction; + } + + static HandlerConfig empty() { return new HandlerConfigBuilderImpl().build(); } + } + private static class RequestContextImpl implements RestApi.RequestContext { final HttpRequest request; final Path pathMatcher; @@ -332,12 +377,14 @@ class RestApiImpl implements RestApi { final Headers headers = new HeadersImpl(); final Attributes attributes = new AttributesImpl(); final RequestContent requestContent; + final AclMapping.Action aclAction; - RequestContextImpl(HttpRequest request, Path pathMatcher, ObjectMapper jacksonJsonMapper) { + RequestContextImpl(HttpRequest request, Path pathMatcher, AclMapping.Action aclAction, ObjectMapper jacksonJsonMapper) { this.request = request; this.pathMatcher = pathMatcher; this.jacksonJsonMapper = jacksonJsonMapper; this.requestContent = request.getData() != null ? new RequestContentImpl() : null; + this.aclAction = aclAction; } @Override public HttpRequest request() { return request; } @@ -357,6 +404,7 @@ class RestApiImpl implements RestApi { ? new UriBuilder(uri.getScheme() + "://" + uri.getHost() + ':' + uriPort) : new UriBuilder(uri.getScheme() + "://" + uri.getHost()); } + @Override public AclMapping.Action aclAction() { return aclAction; } private class PathParametersImpl implements RestApi.RequestContext.PathParameters { @Override @@ -433,63 +481,37 @@ class RestApiImpl implements RestApi { } } - private static class ExceptionMapperHolder<EXCEPTION extends RuntimeException> { - final Class<EXCEPTION> type; - final RestApi.ExceptionMapper<EXCEPTION> mapper; - - ExceptionMapperHolder(Class<EXCEPTION> type, RestApi.ExceptionMapper<EXCEPTION> mapper) { - this.type = type; - this.mapper = mapper; - } - - HttpResponse toResponse(RestApi.RequestContext context, RuntimeException e) { return mapper.toResponse(context, type.cast(e)); } - } - - private static class ResponseMapperHolder<ENTITY> { - final Class<ENTITY> type; - final RestApi.ResponseMapper<ENTITY> mapper; - - ResponseMapperHolder(Class<ENTITY> type, RestApi.ResponseMapper<ENTITY> mapper) { - this.type = type; - this.mapper = mapper; - } - - HttpResponse toHttpResponse(RestApi.RequestContext context, Object entity) { return mapper.toHttpResponse(context, type.cast(entity)); } - } - private static class HandlerHolder<REQUEST_ENTITY> { final Class<REQUEST_ENTITY> type; final HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler; + final HandlerConfig config; - HandlerHolder(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler) { + private HandlerHolder( + Class<REQUEST_ENTITY> type, + HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler, + HandlerConfig config) { this.type = type; this.handler = handler; + this.config = config; } static <RESPONSE_ENTITY, REQUEST_ENTITY> HandlerHolder<REQUEST_ENTITY> of( - Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, RESPONSE_ENTITY> handler) { - return new HandlerHolder<>(type, handler); + Class<REQUEST_ENTITY> type, + HandlerWithRequestEntity<REQUEST_ENTITY, RESPONSE_ENTITY> handler, + HandlerConfig config) { + return new HandlerHolder<>(type, handler, config); } - static <RESPONSE_ENTITY> HandlerHolder<Void> of(Handler<RESPONSE_ENTITY> handler) { + static <RESPONSE_ENTITY> HandlerHolder<Void> of(Handler<RESPONSE_ENTITY> handler, HandlerConfig config) { return new HandlerHolder<>( Void.class, - (HandlerWithRequestEntity<Void, RESPONSE_ENTITY>) (context, nullEntity) -> handler.handleRequest(context)); + (HandlerWithRequestEntity<Void, RESPONSE_ENTITY>) (context, nullEntity) -> handler.handleRequest(context), + config); } Object executeHandler(RestApi.RequestContext context, Object entity) { return handler.handleRequest(context, type.cast(entity)); } } - private static class RequestMapperHolder<ENTITY> { - final Class<ENTITY> type; - final RestApi.RequestMapper<ENTITY> mapper; - - RequestMapperHolder(Class<ENTITY> type, RequestMapper<ENTITY> mapper) { - this.type = type; - this.mapper = mapper; - } - } - static class Route { private final String pathPattern; private final String name; @@ -507,47 +529,10 @@ class RestApiImpl implements RestApi { } private HandlerHolder<?> createDefaultMethodHandler() { - return HandlerHolder.of(context -> { throw new RestApiException.MethodNotAllowed(context.request()); }); + return HandlerHolder.of( + context -> { throw new RestApiException.MethodNotAllowed(context.request()); }, + HandlerConfig.empty()); } } - private static class JacksonRequestMapper<ENTITY> implements RequestMapper<ENTITY> { - private final Class<ENTITY> type; - - JacksonRequestMapper(Class<ENTITY> type) { this.type = type; } - - @Override - public Optional<ENTITY> toRequestEntity(RequestContext context) throws RestApiException { - if (log.isLoggable(Level.FINE)) { - return RestApiImpl.toString(context).map(string -> { - log.fine(() -> "Request content: " + string); - return convertIoException("Failed to parse JSON", () -> context.jacksonJsonMapper().readValue(string, type)); - }); - } else { - return RestApiImpl.toInputStream(context) - .map(in -> convertIoException("Invalid JSON", () -> context.jacksonJsonMapper().readValue(in, type))); - } - } - } - - private static class JacksonResponseMapper<ENTITY> implements ResponseMapper<ENTITY> { - @Override - public HttpResponse toHttpResponse(RequestContext context, ENTITY responseEntity) throws RestApiException { - return new JacksonJsonResponse<>(200, responseEntity, context.jacksonJsonMapper(), true); - } - } - - @FunctionalInterface private interface SupplierThrowingIoException<T> { T get() throws IOException; } - private static <T> T convertIoException(String messagePrefix, SupplierThrowingIoException<T> supplier) { - try { - return supplier.get(); - } catch (IOException e) { - log.log(Level.FINE, e.getMessage(), e); - throw new RestApiException.InternalServerError(messagePrefix + ": " + Exceptions.toMessageString(e), e); - } - } - - private static <T> T convertIoException(SupplierThrowingIoException<T> supplier) { - return convertIoException("Failed to read request content", supplier); - } } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java b/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java new file mode 100644 index 00000000000..36d98421e6a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java @@ -0,0 +1,172 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.restapi; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.RestApi.ExceptionMapper; +import com.yahoo.restapi.RestApi.RequestMapper; +import com.yahoo.restapi.RestApi.ResponseMapper; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Implementations of {@link ExceptionMapper}, {@link RequestMapper} and {@link ResponseMapper}. + * + * @author bjorncs + */ +public class RestApiMappers { + + private static final Logger log = Logger.getLogger(RestApiMappers.class.getName()); + + public static List<RequestMapperHolder<?>> DEFAULT_REQUEST_MAPPERS = List.of( + new RequestMapperHolder<>(Slime.class, RestApiMappers::toSlime), + new RequestMapperHolder<>(JsonNode.class, ctx -> toJsonNode(ctx, ctx.jacksonJsonMapper())), + new RequestMapperHolder<>(String.class, RestApiMappers::toString), + new RequestMapperHolder<>(byte[].class, RestApiMappers::toByteArray), + new RequestMapperHolder<>(InputStream .class, RestApiMappers::toInputStream), + new RequestMapperHolder<>(Void.class, ctx -> Optional.empty())); + + public static List<ResponseMapperHolder<?>> DEFAULT_RESPONSE_MAPPERS = List.of( + new ResponseMapperHolder<>(HttpResponse.class, (context, entity) -> entity), + new ResponseMapperHolder<>(String.class, (context, entity) -> new MessageResponse(entity)), + new ResponseMapperHolder<>(Slime.class, (context, entity) -> new SlimeJsonResponse(entity)), + new ResponseMapperHolder<>(JsonNode.class, + (context, entity) -> new JacksonJsonResponse<>(200, entity, context.jacksonJsonMapper(), true))); + + public static List<ExceptionMapperHolder<?>> DEFAULT_EXCEPTION_MAPPERS = List.of( + new ExceptionMapperHolder<>(RestApiException.class, (context, exception) -> exception.response())); + + private RestApiMappers() {} + + public static class JacksonRequestMapper<ENTITY> implements RequestMapper<ENTITY> { + private final Class<ENTITY> type; + + JacksonRequestMapper(Class<ENTITY> type) { this.type = type; } + + @Override + public Optional<ENTITY> toRequestEntity(RestApi.RequestContext context) throws RestApiException { + if (log.isLoggable(Level.FINE)) { + return RestApiMappers.toString(context).map(string -> { + log.fine(() -> "Request content: " + string); + return convertIoException("Failed to parse JSON", () -> context.jacksonJsonMapper().readValue(string, type)); + }); + } else { + return toInputStream(context) + .map(in -> convertIoException("Invalid JSON", () -> context.jacksonJsonMapper().readValue(in, type))); + } + } + } + + public static class JacksonResponseMapper<ENTITY> implements ResponseMapper<ENTITY> { + @Override + public HttpResponse toHttpResponse(RestApi.RequestContext context, ENTITY responseEntity) throws RestApiException { + return new JacksonJsonResponse<>(200, responseEntity, context.jacksonJsonMapper(), true); + } + } + + public static class RequestMapperHolder<ENTITY> { + final Class<ENTITY> type; + final RestApi.RequestMapper<ENTITY> mapper; + + RequestMapperHolder(Class<ENTITY> type, RequestMapper<ENTITY> mapper) { + this.type = type; + this.mapper = mapper; + } + } + + public static class ResponseMapperHolder<ENTITY> { + final Class<ENTITY> type; + final RestApi.ResponseMapper<ENTITY> mapper; + + ResponseMapperHolder(Class<ENTITY> type, RestApi.ResponseMapper<ENTITY> mapper) { + this.type = type; + this.mapper = mapper; + } + + HttpResponse toHttpResponse(RestApi.RequestContext ctx, Object entity) { + return mapper.toHttpResponse(ctx, type.cast(entity)); + } + } + + public static class ExceptionMapperHolder<EXCEPTION extends RuntimeException> { + final Class<EXCEPTION> type; + final RestApi.ExceptionMapper<EXCEPTION> mapper; + + ExceptionMapperHolder(Class<EXCEPTION> type, RestApi.ExceptionMapper<EXCEPTION> mapper) { + this.type = type; + this.mapper = mapper; + } + + HttpResponse toResponse(RestApi.RequestContext ctx, RuntimeException e) { + return mapper.toResponse(ctx, type.cast(e)); + } + } + + private static Optional<InputStream> toInputStream(RestApi.RequestContext context) { + return context.requestContent().map(RestApi.RequestContext.RequestContent::content); + } + + private static Optional<byte[]> toByteArray(RestApi.RequestContext context) { + InputStream in = toInputStream(context).orElse(null); + if (in == null) return Optional.empty(); + return convertIoException(() -> Optional.of(in.readAllBytes())); + } + + private static Optional<String> toString(RestApi.RequestContext context) { + try { + return toByteArray(context).map(bytes -> new String(bytes, UTF_8)); + } catch (RuntimeException e) { + throw new RestApiException.BadRequest("Failed parse request content as UTF-8: " + Exceptions.toMessageString(e), e); + } + } + + private static Optional<JsonNode> toJsonNode(RestApi.RequestContext context, ObjectMapper jacksonJsonMapper) { + if (log.isLoggable(Level.FINE)) { + return toString(context).map(string -> { + log.fine(() -> "Request content: " + string); + return convertIoException("Failed to parse JSON", () -> jacksonJsonMapper.readTree(string)); + }); + } else { + return toInputStream(context) + .map(in -> convertIoException("Invalid JSON", () -> jacksonJsonMapper.readTree(in))); + } + } + + @FunctionalInterface private interface SupplierThrowingIoException<T> { T get() throws IOException; } + private static <T> T convertIoException(String messagePrefix, SupplierThrowingIoException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.InternalServerError(messagePrefix + ": " + Exceptions.toMessageString(e), e); + } + } + + private static <T> T convertIoException(SupplierThrowingIoException<T> supplier) { + return convertIoException("Failed to read request content", supplier); + } + + private static Optional<Slime> toSlime(RestApi.RequestContext context) { + try { + return toString(context).map(string -> { + log.fine(() -> "Request content: " + string); + return SlimeUtils.jsonToSlimeOrThrow(string); + }); + } catch (com.yahoo.slime.JsonParseException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.BadRequest("Invalid JSON: " + Exceptions.toMessageString(e), e); + } + } + +} diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java index c501ad8c804..4f8adfe9bef 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiRequestHandler.java @@ -4,6 +4,7 @@ package com.yahoo.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.jdisc.Metric; import java.util.concurrent.Executor; @@ -48,6 +49,7 @@ public abstract class RestApiRequestHandler<T extends RestApiRequestHandler<T>> } @Override public final HttpResponse handle(HttpRequest request) { return restApi.handleRequest(request); } + @Override public RequestHandlerSpec requestHandlerSpec() { return restApi.requestHandlerSpec(); } public RestApi restApi() { return restApi; } } diff --git a/container-core/src/main/resources/configdefinitions/container.di.config.jersey-bundles.def b/container-core/src/main/resources/configdefinitions/container.di.config.jersey-bundles.def deleted file mode 100644 index a226420274d..00000000000 --- a/container-core/src/main/resources/configdefinitions/container.di.config.jersey-bundles.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.di.config - -# The SymbolicName[:Version] of the Jersey bundles -bundles[].spec string - -# The packages to scan for Jersey resources -bundles[].packages[] string diff --git a/container-core/src/main/resources/configdefinitions/container.di.config.jersey-injection.def b/container-core/src/main/resources/configdefinitions/container.di.config.jersey-injection.def deleted file mode 100644 index 9f5be59abbd..00000000000 --- a/container-core/src/main/resources/configdefinitions/container.di.config.jersey-injection.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.di.config - -inject[].instance string -inject[].forClass string diff --git a/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java b/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java index b596246a43d..47ac3018fde 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java @@ -5,13 +5,10 @@ import com.google.inject.Guice; import com.yahoo.component.AbstractComponent; import com.yahoo.config.di.IntConfig; import com.yahoo.config.test.TestConfig; -import com.yahoo.container.bundle.MockBundle; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.di.componentgraph.core.ComponentGraph; import com.yahoo.container.di.componentgraph.core.ComponentGraphTest.SimpleComponent; -import com.yahoo.container.di.componentgraph.core.ComponentGraphTest.SimpleComponent2; import com.yahoo.container.di.componentgraph.core.ComponentNode.ComponentConstructorException; -import com.yahoo.container.di.config.RestApiContext; import org.junit.Ignore; import org.junit.Test; import org.osgi.framework.Bundle; @@ -218,64 +215,6 @@ public class ContainerTest extends ContainerTestBase { assertNotNull(newGraph.get(5, TimeUnit.MINUTES)); } - - @Test - public void bundle_info_is_set_on_rest_api_context() { - Class<RestApiContext> clazz = RestApiContext.class; - - writeBootstrapConfigs("restApiContext", clazz); - dirConfigSource.writeConfig("jersey-bundles", "bundles[0].spec \"mock-entry-to-enforce-a-MockBundle\""); - dirConfigSource.writeConfig("jersey-injection", "inject[0]"); - - Container container = newContainer(dirConfigSource); - ComponentGraph componentGraph = getNewComponentGraph(container); - - RestApiContext restApiContext = componentGraph.getInstance(clazz); - assertNotNull(restApiContext); - - assertEquals(1, restApiContext.getBundles().size()); - assertEquals(MockBundle.SymbolicName, restApiContext.getBundles().get(0).symbolicName); - assertEquals(MockBundle.BundleVersion, restApiContext.getBundles().get(0).version); - - container.shutdownConfigurer(); - } - - @Test - public void restApiContext_has_all_components_injected() { - Class<RestApiContext> restApiClass = RestApiContext.class; - Class<SimpleComponent> injectedClass = SimpleComponent.class; - String injectedComponentId = "injectedComponent"; - Class<SimpleComponent2> anotherComponentClass = SimpleComponent2.class; - String anotherComponentId = "anotherComponent"; - - String componentsConfig = - new ComponentEntry(injectedComponentId, injectedClass).asConfig(0) + "\n" + - new ComponentEntry(anotherComponentId, anotherComponentClass).asConfig(1) + "\n" + - new ComponentEntry("restApiContext", restApiClass).asConfig(2) + "\n" + - "components[2].inject[0].id " + injectedComponentId + "\n" + - "components[2].inject[1].id " + anotherComponentId + "\n"; - - String injectionConfig = "inject[1]\n" +// - "inject[0].instance " + injectedComponentId + "\n" +// - "inject[0].forClass \"" + injectedClass.getName() + "\"\n"; - - dirConfigSource.writeConfig("components", componentsConfig); - dirConfigSource.writeConfig("platform-bundles", ""); - dirConfigSource.writeConfig("application-bundles", ""); - dirConfigSource.writeConfig("jersey-bundles", "bundles[0].spec \"mock-entry-to-enforce-a-MockBundle\""); - dirConfigSource.writeConfig("jersey-injection", injectionConfig); - - Container container = newContainer(dirConfigSource); - ComponentGraph componentGraph = getNewComponentGraph(container); - - RestApiContext restApiContext = componentGraph.getInstance(restApiClass); - - assertFalse(restApiContext.getInjectableComponents().isEmpty()); - assertEquals(2, restApiContext.getInjectableComponents().size()); - - container.shutdownConfigurer(); - } - @Test public void providers_are_destructed() { writeBootstrapConfigs("id1", DestructableProvider.class); diff --git a/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java b/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java index 2106a1f3671..0d1a308f182 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/di/ContainerTestBase.java @@ -7,7 +7,6 @@ import com.yahoo.config.FileReference; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.di.ContainerTest.ComponentTakingConfig; import com.yahoo.container.di.componentgraph.core.ComponentGraph; -import com.yahoo.container.di.osgi.BundleClasses; import org.junit.After; import org.junit.Before; import org.osgi.framework.Bundle; @@ -57,11 +56,6 @@ public class ContainerTestBase { } @Override - public BundleClasses getBundleClasses(ComponentSpecification bundle, Set<String> packagesToScan) { - throw new UnsupportedOperationException("getBundleClasses not supported"); - } - - @Override public Set<Bundle> useApplicationBundles(Collection<FileReference> bundles) { return emptySet(); } diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index 70dc4c8665c..43e36781101 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -16,11 +16,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.test.Test2Config; import com.yahoo.config.test.TestConfig; -import com.yahoo.container.di.Osgi; import com.yahoo.container.di.componentgraph.Provider; -import com.yahoo.container.di.config.JerseyBundlesConfig; -import com.yahoo.container.di.config.JerseyInjectionConfig; -import com.yahoo.container.di.config.RestApiContext; import com.yahoo.vespa.config.ConfigKey; import org.junit.Test; @@ -485,25 +481,6 @@ public class ComponentGraphTest { assertThat(componentGraph.getInstance(ComponentTakingComponentId.class).componentId, is(ComponentId.fromString(componentId))); } - @Test - public void rest_api_context_can_be_instantiated() { - String configId = "raw:\"\""; - - Class<RestApiContext> clazz = RestApiContext.class; - JerseyNode jerseyNode = new JerseyNode(uniqueComponentId(clazz.getName()), configId, clazz, new Osgi() { - }); - - ComponentGraph componentGraph = new ComponentGraph(); - componentGraph.add(jerseyNode); - componentGraph.complete(); - - componentGraph - .setAvailableConfigs(ConfigMap.newMap(JerseyBundlesConfig.class, configId).add(JerseyInjectionConfig.class, configId)); - - RestApiContext restApiContext = componentGraph.getInstance(clazz); - assertNotNull(restApiContext); - assertThat(restApiContext.getBundles().size(), is(0)); - } //Note that all Components must be defined in a static context, //otherwise their constructor will take the outer class as the first parameter. diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/JerseyNodeTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/JerseyNodeTest.java deleted file mode 100644 index f30f9260830..00000000000 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/JerseyNodeTest.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.di.componentgraph.core; - -import com.yahoo.container.bundle.MockBundle; -import com.yahoo.container.di.config.RestApiContext; -import com.yahoo.container.di.osgi.OsgiUtil; -import org.junit.Test; -import org.osgi.framework.wiring.BundleWiring; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.assertThat; - -/** - * @author gjoranv - * @author ollivir - */ - -public class JerseyNodeTest { - private MockBundle bundle; - private List<String> bundleClasses; - private final Map<String, String> resources; - - public JerseyNodeTest() { - resources = new HashMap<>(); - resources.put("com/foo", "com/foo/Foo.class"); - resources.put("com/bar", "com/bar/Bar.class"); - bundle = new MockBundle() { - @Override - public Collection<String> listResources(String path, String ignored, int options) { - if ((options & BundleWiring.LISTRESOURCES_RECURSE) != 0 && path.equals("/")) { - return resources.values(); - } else { - return Collections.singleton(resources.get(path)); - } - } - }; - bundleClasses = new ArrayList<>(resources.values()); - } - - @Test - public void all_bundle_entries_are_returned_when_no_packages_are_given() { - Collection<String> entries = OsgiUtil.getClassEntriesInBundleClassPath(bundle, Collections.emptySet()); - assertThat(entries, containsInAnyOrder(bundleClasses.toArray())); - } - - @Test - public void only_bundle_entries_from_the_given_packages_are_returned() { - Collection<String> entries = OsgiUtil.getClassEntriesInBundleClassPath(bundle, Collections.singleton("com.foo")); - assertThat(entries, contains(resources.get("com/foo"))); - } - - @Test - public void bundle_info_is_initialized() { - RestApiContext.BundleInfo bundleInfo = JerseyNode.createBundleInfo(bundle, Collections.emptyList()); - assertThat(bundleInfo.symbolicName, is(bundle.getSymbolicName())); - assertThat(bundleInfo.version, is(bundle.getVersion())); - assertThat(bundleInfo.fileLocation, is(bundle.getLocation())); - } -} diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index bd3000a0188..86a3808becc 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -701,6 +701,28 @@ public class HttpServerTest { .set(MetricDefinitions.REQUESTS_PER_CONNECTION, 1L, MetricConsumerMock.STATIC_CONTEXT); } + @Test + public void uriWithEmptyPathSegmentIsAllowed() throws Exception { + Path privateKeyFile = tmpFolder.newFile().toPath(); + Path certificateFile = tmpFolder.newFile().toPath(); + generatePrivateKeyAndCertificate(privateKeyFile, certificateFile); + MetricConsumerMock metricConsumer = new MetricConsumerMock(); + InMemoryConnectionLog connectionLog = new InMemoryConnectionLog(); + JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog); + String uriPath = "/path/with/empty//segment"; + + // HTTP/1.1 + driver.client().get(uriPath).expectStatusCode(is(OK)); + + // HTTP/2 + try (CloseableHttpAsyncClient client = createHttp2Client(driver)) { + String uri = "https://localhost:" + driver.server().getListenPort() + uriPath; + SimpleHttpResponse response = client.execute(SimpleRequestBuilder.get(uri).build(), null).get(); + assertEquals(OK, response.getCode()); + } + + assertTrue(driver.close()); + } private static CloseableHttpAsyncClient createHttp2Client(JettyTestDriver driver) { TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() diff --git a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java index 06fc6d80741..44dd61836d6 100644 --- a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java +++ b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java @@ -1,14 +1,18 @@ package com.yahoo.restapi;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.RequestHandlerSpec; +import com.yahoo.container.jdisc.RequestView; import com.yahoo.test.json.JsonTestHelper; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; @@ -16,6 +20,7 @@ import java.util.List; import java.util.Map; import static com.yahoo.jdisc.http.HttpRequest.Method; +import static com.yahoo.restapi.RestApi.handlerConfig; import static com.yahoo.restapi.RestApi.route; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -103,13 +108,32 @@ class RestApiImplTest { } @Test - public void uri_builder_creates_valid_uri_prefix() { + void uri_builder_creates_valid_uri_prefix() { RestApi restApi = RestApi.builder() .addRoute(route("/test").get(ctx -> new MessageResponse(ctx.uriBuilder().toString()))) .build(); verifyJsonResponse(restApi, Method.GET, "/test", null, 200, "{\"message\":\"http://localhost\"}"); } + @Test + void resolves_correct_acl_action() { + AclMapping.Action customAclAction = AclMapping.Action.custom("custom-action"); + RestApi restApi = RestApi.builder() + .addRoute(route("/api1") + .get(ctx -> new MessageResponse(ctx.aclAction().name()), + handlerConfig().withCustomAclAction(customAclAction))) + .addRoute(route("/api2") + .post(ctx -> new MessageResponse(ctx.aclAction().name()))) + .build(); + + verifyJsonResponse(restApi, Method.GET, "/api1", null, 200, "{\"message\":\"custom-action\"}"); + verifyJsonResponse(restApi, Method.POST, "/api2", "ignored", 200, "{\"message\":\"write\"}"); + + RequestHandlerSpec spec = restApi.requestHandlerSpec(); + assertRequestHandlerSpecAclMapping(spec, customAclAction, Method.GET, "/api1"); + assertRequestHandlerSpecAclMapping(spec, AclMapping.Action.WRITE, Method.POST, "/api2"); + } + private static void verifyJsonResponse(RestApi restApi, Method method, String path, String requestContent, int expectedStatusCode, String expectedJson) { HttpRequest testRequest; String uri = "http://localhost" + path; @@ -132,6 +156,15 @@ class RestApiImplTest { } } + private static void assertRequestHandlerSpecAclMapping( + RequestHandlerSpec spec, AclMapping.Action expectedAction, Method method, String uriPath) { + RequestView requestView = new RequestView() { + @Override public Method method() { return method; } + @Override public URI uri() { return URI.create("http://localhost" + uriPath); } + }; + assertEquals(expectedAction, spec.aclMapping().get(requestView)); + } + public static class TestEntity { @JsonProperty("mystring") public String stringValue; @JsonProperty("myinstant") public Instant instantValue; diff --git a/container-disc/src/main/java/com/yahoo/container/config/jersey/package-info.java b/container-disc/src/main/java/com/yahoo/container/config/jersey/package-info.java deleted file mode 100644 index 739bc118be6..00000000000 --- a/container-disc/src/main/java/com/yahoo/container/config/jersey/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.container.config.jersey; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-disc/src/main/resources/configdefinitions/container.config.jersey.jersey-connection.def b/container-disc/src/main/resources/configdefinitions/container.config.jersey.jersey-connection.def deleted file mode 100644 index b8662e07bf9..00000000000 --- a/container-disc/src/main/resources/configdefinitions/container.config.jersey.jersey-connection.def +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -## Do NOT move this file to the container-jersey module. If system bundles -## like config-model import packages from container-jersey, new class -## loaders for these bundles will be created after reconfig. -namespace=container.config.jersey - -requestTimeout long default=-1 diff --git a/container-disc/src/main/resources/configdefinitions/container.config.jersey.jersey-web-app-pool.def b/container-disc/src/main/resources/configdefinitions/container.config.jersey.jersey-web-app-pool.def deleted file mode 100644 index 52cb9f2229a..00000000000 --- a/container-disc/src/main/resources/configdefinitions/container.config.jersey.jersey-web-app-pool.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -## Do NOT move this file to the container-jersey module. If system bundles -## like config-model import packages from container-jersey, new class -## loaders for these bundles will be created after reconfig. -namespace=container.config.jersey - -maxActiveWebApp int default=4 -maxIdleWebApp int default=4 diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 5f46b949844..9b63f5630c9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; @@ -9,26 +8,29 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeHistory; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; /** * A node in hosted Vespa. * + * This is immutable and all fields are guaranteed to be non-null. This should never leak any wire format types or + * types from third-party libraries. + * + * Use {@link Node#builder()} or {@link Node#builder(Node)} to create instances of this. + * * @author mpolden * @author jonmv */ public class Node { + private final String id; private final HostName hostname; private final Optional<HostName> parentHostname; private final State state; @@ -50,204 +52,278 @@ public class Node { private final long rebootGeneration; private final long wantedRebootGeneration; private final int cost; + private final int failCount; private final String flavor; private final String clusterId; private final ClusterType clusterType; + private final String group; private final boolean retired; private final boolean wantToRetire; private final boolean wantToDeprovision; private final boolean wantToRebuild; private final Optional<TenantName> reservedTo; private final Optional<ApplicationId> exclusiveTo; - private final Map<String, JsonNode> reports; - private final List<NodeHistory> history; + private final Map<String, String> reports; + private final List<Event> history; + private final Set<String> ipAddresses; private final Set<String> additionalIpAddresses; - private final String openStackId; + private final Set<String> additionalHostnames; private final Optional<String> switchHostname; private final Optional<String> modelName; - - public Node(HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, NodeResources resources, Optional<ApplicationId> owner, - Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, - Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, - Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, - int cost, String flavor, String clusterId, ClusterType clusterType, boolean retired, boolean wantToRetire, boolean wantToDeprovision, - boolean wantToRebuild, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, - DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, JsonNode> reports, List<NodeHistory> history, - Set<String> additionalIpAddresses, String openStackId, Optional<String> switchHostname, Optional<String> modelName) { - this.hostname = hostname; - this.parentHostname = parentHostname; - this.state = state; - this.type = type; - this.resources = resources; - this.owner = owner; - this.currentVersion = currentVersion; - this.wantedVersion = wantedVersion; - this.currentOsVersion = currentOsVersion; - this.wantedOsVersion = wantedOsVersion; - this.currentFirmwareCheck = currentFirmwareCheck; - this.wantedFirmwareCheck = wantedFirmwareCheck; - this.serviceState = serviceState; - this.suspendedSince = suspendedSince; + private final Environment environment; + + private Node(String id, HostName hostname, Optional<HostName> parentHostname, State state, NodeType type, + NodeResources resources, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, + Version currentOsVersion, Version wantedOsVersion, Optional<Instant> currentFirmwareCheck, + Optional<Instant> wantedFirmwareCheck, ServiceState serviceState, Optional<Instant> suspendedSince, + long restartGeneration, long wantedRestartGeneration, long rebootGeneration, + long wantedRebootGeneration, int cost, int failCount, String flavor, String clusterId, + ClusterType clusterType, String group, boolean retired, boolean wantToRetire, boolean wantToDeprovision, + boolean wantToRebuild, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveTo, + DockerImage wantedDockerImage, DockerImage currentDockerImage, Map<String, String> reports, + List<Event> history, Set<String> ipAddresses, Set<String> additionalIpAddresses, + Set<String> additionalHostnames, Optional<String> switchHostname, + Optional<String> modelName, Environment environment) { + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.parentHostname = Objects.requireNonNull(parentHostname, "parentHostname must be non-null"); + this.state = Objects.requireNonNull(state, "state must be non-null"); + this.type = Objects.requireNonNull(type, "type must be non-null"); + this.resources = Objects.requireNonNull(resources, "resources must be non-null"); + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.currentVersion = Objects.requireNonNull(currentVersion, "currentVersion must be non-null"); + this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); + this.currentOsVersion = Objects.requireNonNull(currentOsVersion, "currentOsVersion must be non-null"); + this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion, "wantedOsVersion must be non-null"); + this.currentFirmwareCheck = Objects.requireNonNull(currentFirmwareCheck, "currentFirmwareCheck must be non-null"); + this.wantedFirmwareCheck = Objects.requireNonNull(wantedFirmwareCheck, "wantedFirmwareCheck must be non-null"); + this.serviceState = Objects.requireNonNull(serviceState, "serviceState must be non-null"); + this.suspendedSince = Objects.requireNonNull(suspendedSince, "suspendedSince must be non-null"); this.restartGeneration = restartGeneration; this.wantedRestartGeneration = wantedRestartGeneration; this.rebootGeneration = rebootGeneration; this.wantedRebootGeneration = wantedRebootGeneration; this.cost = cost; - this.flavor = flavor; - this.clusterId = clusterId; - this.clusterType = clusterType; + this.failCount = failCount; + this.flavor = Objects.requireNonNull(flavor, "flavor must be non-null"); + this.clusterId = Objects.requireNonNull(clusterId, "clusterId must be non-null"); + this.clusterType = Objects.requireNonNull(clusterType, "clusterType must be non-null"); this.retired = retired; + this.group = Objects.requireNonNull(group, "group must be non-null"); this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; - this.reservedTo = reservedTo; - this.exclusiveTo = exclusiveTo; - this.wantedDockerImage = wantedDockerImage; - this.currentDockerImage = currentDockerImage; + this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo must be non-null"); + this.exclusiveTo = Objects.requireNonNull(exclusiveTo, "exclusiveTo must be non-null"); + this.wantedDockerImage = Objects.requireNonNull(wantedDockerImage, "wantedDockerImage must be non-null"); + this.currentDockerImage = Objects.requireNonNull(currentDockerImage, "currentDockerImage must be non-null"); this.wantToRebuild = wantToRebuild; - this.reports = reports; - this.history = history; - this.openStackId = openStackId; - this.additionalIpAddresses = additionalIpAddresses; - this.switchHostname = switchHostname; - this.modelName = modelName; + this.reports = Map.copyOf(Objects.requireNonNull(reports, "reports must be non-null")); + this.history = List.copyOf(Objects.requireNonNull(history, "history must be non-null")); + this.ipAddresses = Set.copyOf(Objects.requireNonNull(ipAddresses, "ipAddresses must be non-null")); + this.additionalIpAddresses = Set.copyOf(Objects.requireNonNull(additionalIpAddresses, "additionalIpAddresses must be non-null")); + this.additionalHostnames = Set.copyOf(Objects.requireNonNull(additionalHostnames, "additionalHostnames must be non-null")); + this.switchHostname = Objects.requireNonNull(switchHostname, "switchHostname must be non-null"); + this.modelName = Objects.requireNonNull(modelName, "modelName must be non-null"); + this.environment = Objects.requireNonNull(environment, "environment must be non-ull"); + } + + /** The cloud provider's unique ID for this */ + public String id() { + return id; } + /** The hostname of this */ public HostName hostname() { return hostname; } + /** The parent hostname of this, if any */ public Optional<HostName> parentHostname() { return parentHostname; } + /** Current state of this */ public State state() { return state; } + /** The node type of this */ public NodeType type() { return type; } + /** Resources, such as CPU and memory, of this */ public NodeResources resources() { return resources; } + /** The application owning this, if any */ public Optional<ApplicationId> owner() { return owner; } + /** The Vespa version this is currently running */ public Version currentVersion() { return currentVersion; } + /** The wanted Vespa version */ public Version wantedVersion() { return wantedVersion; } + /** The OS version this is currently running */ public Version currentOsVersion() { return currentOsVersion; } + /** The wanted OS version */ public Version wantedOsVersion() { return wantedOsVersion; } + /** The container image of this is currently running */ public DockerImage currentDockerImage() { return currentDockerImage; } + /** The wanted Docker image */ public DockerImage wantedDockerImage() { return wantedDockerImage; } + /** The last time this checked for a firmware update */ public Optional<Instant> currentFirmwareCheck() { return currentFirmwareCheck; } + /** The wanted time this should check for a firmware update */ public Optional<Instant> wantedFirmwareCheck() { return wantedFirmwareCheck; } + /** The current service state of this */ public ServiceState serviceState() { return serviceState; } + /** The most recent time this suspended, if any */ public Optional<Instant> suspendedSince() { return suspendedSince; } + /** The current restart generation */ public long restartGeneration() { return restartGeneration; } + /** The wanted restart generation */ public long wantedRestartGeneration() { return wantedRestartGeneration; } + /** The current reboot generation */ public long rebootGeneration() { return rebootGeneration; } + /** The wanted reboot generation */ public long wantedRebootGeneration() { return wantedRebootGeneration; } + /** A number representing the cost of this */ public int cost() { return cost; } + /** How many times this has failed */ + public int failCount() { + return failCount; + } + + /** The flavor of this */ public String flavor() { return flavor; } + /** The cluster ID of this, empty string if unallocated */ public String clusterId() { return clusterId; } + /** The cluster type of this */ public ClusterType clusterType() { return clusterType; } + /** Whether this is retired */ public boolean retired() { return retired; } + /** The group of this node, empty string if unallocated */ + public String group() { + return group; + } + + /** Whether this node has been requested to retire */ public boolean wantToRetire() { return wantToRetire; } + /** Whether this node has been requested to deprovision */ public boolean wantToDeprovision() { return wantToDeprovision; } + /** Whether this node has been requested to rebuild */ public boolean wantToRebuild() { return wantToRebuild; } + /** The tenant this has been reserved to, if any */ public Optional<TenantName> reservedTo() { return reservedTo; } + /** The application this has been provisioned exclusively for, if any */ public Optional<ApplicationId> exclusiveTo() { return exclusiveTo; } - public Map<String, JsonNode> reports() { + /** Returns the reports of this node. Key is the report ID. Value is untyped, but is typically a JSON string */ + public Map<String, String> reports() { return reports; } - public List<NodeHistory> history() { + /** History of events affecting this */ + public List<Event> history() { return history; } + /** IP addresses of this */ + public Set<String> ipAddresses() { + return ipAddresses; + } + + /** Additional IP addresses available on this, usable by child nodes */ public Set<String> additionalIpAddresses() { return additionalIpAddresses; } - public String openStackId() { - return openStackId; + /** Additional hostnames available on this, usable by child nodes */ + public Set<String> additionalHostnames() { + return additionalHostnames; } + /** Hostname of the switch this is connected to, if any */ public Optional<String> switchHostname() { return switchHostname; } + /** The server model of this, if any */ public Optional<String> modelName() { return modelName; } + /** The environment this runs in */ + public Environment environment() { + return environment; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -293,48 +369,118 @@ public class Node { combined, unknown } + + /** Known nope environments */ + public enum Environment { + bareMetal, + virtualMachine, + dockerContainer, + } + + /** A node event */ + public static class Event { + + private final Instant at; + private final String agent; + private final String name; + + public Event(Instant at, String agent, String name) { + this.at = Objects.requireNonNull(at); + this.agent = Objects.requireNonNull(agent); + this.name = Objects.requireNonNull(name); + } + + /** The time this occurred */ + public Instant at() { + return at; + } + + /** The agent responsible for this */ + public String agent() { + return agent; + } + + /** Name of the event */ + public String name() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Event event = (Event) o; + return at.equals(event.at) && agent.equals(event.agent) && name.equals(event.name); + } + + @Override + public int hashCode() { + return Objects.hash(at, agent, name); + } + } + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(Node node) { + return new Builder(node); + } + + /** + * Builder for a {@link Node}. + * + * The appropriate builder method must be called for any field that does not have a default value. + */ public static class Builder { + private HostName hostname; + + private String id = UUID.randomUUID().toString(); private Optional<HostName> parentHostname = Optional.empty(); - private State state; - private NodeType type; - private NodeResources resources; + private State state = State.active; + private NodeType type = NodeType.host; + private NodeResources resources = NodeResources.unspecified(); private Optional<ApplicationId> owner = Optional.empty(); - private Version currentVersion; - private Version wantedVersion; - private Version currentOsVersion; - private Version wantedOsVersion; - private DockerImage currentDockerImage; - private DockerImage wantedDockerImage; + private Version currentVersion = Version.emptyVersion; + private Version wantedVersion = Version.emptyVersion; + private Version currentOsVersion = Version.emptyVersion; + private Version wantedOsVersion = Version.emptyVersion; + private DockerImage currentDockerImage = DockerImage.EMPTY; + private DockerImage wantedDockerImage = DockerImage.EMPTY; private Optional<Instant> currentFirmwareCheck = Optional.empty(); private Optional<Instant> wantedFirmwareCheck = Optional.empty(); - private ServiceState serviceState; + private ServiceState serviceState = ServiceState.unknown; private Optional<Instant> suspendedSince = Optional.empty(); - private long restartGeneration; - private long wantedRestartGeneration; - private long rebootGeneration; - private long wantedRebootGeneration; - private int cost; - private String flavor; - private String clusterId; - private ClusterType clusterType; - private boolean retired; - private boolean wantToRetire; - private boolean wantToDeprovision; - private boolean wantToRebuild; + private long restartGeneration = 0; + private long wantedRestartGeneration = 0; + private long rebootGeneration = 0; + private long wantedRebootGeneration = 0; + private int cost = 0; + private int failCount = 0; + private String flavor = "default"; + private String clusterId = ""; + private ClusterType clusterType = ClusterType.unknown; + private String group = ""; + private boolean retired = false; + private boolean wantToRetire = false; + private boolean wantToDeprovision = false; + private boolean wantToRebuild = false; private Optional<TenantName> reservedTo = Optional.empty(); private Optional<ApplicationId> exclusiveTo = Optional.empty(); - private Map<String, JsonNode> reports = new HashMap<>(); - private List<NodeHistory> history = new ArrayList<>(); - private Set<String> additionalIpAddresses = new HashSet<>(); - private String openStackId; + private Map<String, String> reports = Map.of(); + private List<Event> history = List.of(); + private Set<String> ipAddresses = Set.of(); + private Set<String> additionalIpAddresses = Set.of(); + private Set<String> additionalHostnames = Set.of(); private Optional<String> switchHostname = Optional.empty(); private Optional<String> modelName = Optional.empty(); + private Environment environment = Environment.bareMetal; - public Builder() { } + private Builder() {} - public Builder(Node node) { + private Builder(Node node) { + this.id = node.id; this.hostname = node.hostname; this.parentHostname = node.parentHostname; this.state = node.state; @@ -347,18 +493,20 @@ public class Node { this.wantedOsVersion = node.wantedOsVersion; this.currentDockerImage = node.currentDockerImage; this.wantedDockerImage = node.wantedDockerImage; - this.currentFirmwareCheck = node.currentFirmwareCheck; - this.wantedFirmwareCheck = node.wantedFirmwareCheck; this.serviceState = node.serviceState; this.suspendedSince = node.suspendedSince; + this.currentFirmwareCheck = node.currentFirmwareCheck; + this.wantedFirmwareCheck = node.wantedFirmwareCheck; this.restartGeneration = node.restartGeneration; this.wantedRestartGeneration = node.wantedRestartGeneration; this.rebootGeneration = node.rebootGeneration; this.wantedRebootGeneration = node.wantedRebootGeneration; this.cost = node.cost; + this.failCount = node.failCount; this.flavor = node.flavor; this.clusterId = node.clusterId; this.clusterType = node.clusterType; + this.group = node.group; this.retired = node.retired; this.wantToRetire = node.wantToRetire; this.wantToDeprovision = node.wantToDeprovision; @@ -367,10 +515,21 @@ public class Node { this.exclusiveTo = node.exclusiveTo; this.reports = node.reports; this.history = node.history; + this.ipAddresses = node.ipAddresses; this.additionalIpAddresses = node.additionalIpAddresses; - this.openStackId = node.openStackId; + this.additionalHostnames = node.additionalHostnames; this.switchHostname = node.switchHostname; this.modelName = node.modelName; + this.environment = node.environment; + } + + public Builder id(String id) { + this.id = id; + return this; + } + + public Builder hostname(String hostname) { + return hostname(HostName.from(hostname)); } public Builder hostname(HostName hostname) { @@ -378,6 +537,10 @@ public class Node { return this; } + public Builder parentHostname(String parentHostname) { + return parentHostname(HostName.from(parentHostname)); + } + public Builder parentHostname(HostName parentHostname) { this.parentHostname = Optional.ofNullable(parentHostname); return this; @@ -478,6 +641,11 @@ public class Node { return this; } + public Builder failCount(int failCount) { + this.failCount = failCount; + return this; + } + public Builder flavor(String flavor) { this.flavor = flavor; return this; @@ -493,6 +661,11 @@ public class Node { return this; } + public Builder group(String group) { + this.group = group; + return this; + } + public Builder retired(boolean retired) { this.retired = retired; return this; @@ -523,18 +696,23 @@ public class Node { return this; } - public Builder history(List<NodeHistory> history) { + public Builder history(List<Event> history) { this.history = history; return this; } + public Builder ipAddresses(Set<String> ipAdresses) { + this.ipAddresses = ipAdresses; + return this; + } + public Builder additionalIpAddresses(Set<String> additionalIpAddresses) { this.additionalIpAddresses = additionalIpAddresses; return this; } - public Builder openStackId(String openStackId) { - this.openStackId = openStackId; + public Builder additionalHostnames(Set<String> additionalHostnames) { + this.additionalHostnames = additionalHostnames; return this; } @@ -548,18 +726,26 @@ public class Node { return this; } - public Builder reports(Map<String, JsonNode> reports) { + public Builder reports(Map<String, String> reports) { this.reports = reports; return this; } + public Builder environment(Environment environment) { + this.environment = environment; + return this; + } + public Node build() { - return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, + return new Node(id, hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion, currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState, - suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration, - cost, flavor, clusterId, clusterType, retired, wantToRetire, wantToDeprovision, wantToRebuild, reservedTo, exclusiveTo, - wantedDockerImage, currentDockerImage, reports, history, additionalIpAddresses, openStackId, switchHostname, modelName); + suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, + wantedRebootGeneration, cost, failCount, flavor, clusterId, clusterType, group, retired, + wantToRetire, wantToDeprovision, wantToRebuild, reservedTo, exclusiveTo, wantedDockerImage, + currentDockerImage, reports, history, ipAddresses, additionalIpAddresses, + additionalHostnames, switchHostname, modelName, environment); } } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index ac4ff0a80a0..3c16eac06c7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -3,21 +3,15 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.OrchestratorStatus; import java.net.URI; import java.time.Duration; -import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; @@ -26,7 +20,7 @@ import java.util.Set; import java.util.stream.Collectors; /** - * A minimal interface to the node repository, providing only the operations used by the controller. + * Node repository interface intended for use by the controller. * * @author mpolden */ @@ -38,10 +32,7 @@ public interface NodeRepository { void setState(ZoneId zone, NodeState nodeState, String hostname); - NodeRepositoryNode getNode(ZoneId zone, String hostname); - - // TODO: Migrate any callers to list() and remove this method - NodeList listNodes(ZoneId zone); + Node getNode(ZoneId zone, String hostname); /** List all nodes in given zone */ List<Node> list(ZoneId zone, boolean includeDeprovisioned); @@ -96,145 +87,4 @@ public interface NodeRepository { /** Checks whether the zone has the spare capacity to remove the given hosts */ boolean isReplaceable(ZoneId zoneId, List<HostName> hostNames); - static Node toNode(NodeRepositoryNode node) { - var application = Optional.ofNullable(node.getOwner()) - .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(), - owner.getInstance())); - var parentHostname = Optional.ofNullable(node.getParentHostname()).map(HostName::from); - var resources = new NodeResources( - toDouble(node.getResources().getVcpu()), - toDouble(node.getResources().getMemoryGb()), - toDouble(node.getResources().getDiskGb()), - toDouble(node.getResources().getBandwidthGbps()), - diskSpeedFromString(node.getResources().getDiskSpeed()), - storageTypeFromString(node.getResources().getStorageType())); - return new Node(HostName.from(node.getHostname()), - parentHostname, - fromJacksonState(node.getState()), - fromJacksonType(node.getType()), - resources, - application, - versionFrom(node.getVespaVersion()), - versionFrom(node.getWantedVespaVersion()), - versionFrom(node.getCurrentOsVersion()), - versionFrom(node.getWantedOsVersion()), - Optional.ofNullable(node.getCurrentFirmwareCheck()).map(Instant::ofEpochMilli), - Optional.ofNullable(node.getWantedFirmwareCheck()).map(Instant::ofEpochMilli), - toServiceState(node.getOrchestratorStatus()), - Optional.ofNullable(node.suspendedSinceMillis()).map(Instant::ofEpochMilli), - toInt(node.getCurrentRestartGeneration()), - toInt(node.getRestartGeneration()), - toInt(node.getCurrentRebootGeneration()), - toInt(node.getRebootGeneration()), - toInt(node.getCost()), - node.getFlavor(), - clusterIdOf(node.getMembership()), - clusterTypeOf(node.getMembership()), - Optional.ofNullable(node.getMembership()).map(NodeMembership::getRetired).orElse(false), - node.getWantToRetire(), - node.getWantToDeprovision(), - node.getWantToRebuild(), Optional.ofNullable(node.getReservedTo()).map(TenantName::from), - Optional.ofNullable(node.getExclusiveTo()).map(ApplicationId::fromSerializedForm), - dockerImageFrom(node.getWantedDockerImage()), - dockerImageFrom(node.getCurrentDockerImage()), - node.getReports(), - node.getHistory(), - node.getAdditionalIpAddresses(), - node.getOpenStackId(), - Optional.ofNullable(node.getSwitchHostname()), - Optional.ofNullable(node.getModelName())); - } - - private static String clusterIdOf(NodeMembership nodeMembership) { - return nodeMembership == null ? "" : nodeMembership.clusterid; - } - - private static Node.ClusterType clusterTypeOf(NodeMembership nodeMembership) { - if (nodeMembership == null) return Node.ClusterType.unknown; - switch (nodeMembership.clustertype) { - case "admin": return Node.ClusterType.admin; - case "content": return Node.ClusterType.content; - case "container": return Node.ClusterType.container; - case "combined": return Node.ClusterType.combined; - } - return Node.ClusterType.unknown; - } - - // Convert Jackson type to config.provision type - private static NodeType fromJacksonType(com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType nodeType) { - switch (nodeType) { - case tenant: return NodeType.tenant; - case host: return NodeType.host; - case proxy: return NodeType.proxy; - case proxyhost: return NodeType.proxyhost; - case config: return NodeType.config; - case confighost: return NodeType.confighost; - case controller: return NodeType.controller; - case controllerhost: return NodeType.controllerhost; - default: throw new IllegalArgumentException("Unknown type: " + nodeType); - } - } - - private static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State fromJacksonState(NodeState state) { - switch (state) { - case provisioned: return Node.State.provisioned; - case ready: return Node.State.ready; - case reserved: return Node.State.reserved; - case active: return Node.State.active; - case inactive: return Node.State.inactive; - case dirty: return Node.State.dirty; - case failed: return Node.State.failed; - case parked: return Node.State.parked; - case breakfixed: return Node.State.breakfixed; - case deprovisioned: return Node.State.deprovisioned; - } - return Node.State.unknown; - } - - private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) { - if (diskSpeed == null) return NodeResources.DiskSpeed.getDefault(); - switch (diskSpeed) { - case "fast": return NodeResources.DiskSpeed.fast; - case "slow": return NodeResources.DiskSpeed.slow; - case "any": return NodeResources.DiskSpeed.any; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); - } - } - - private static NodeResources.StorageType storageTypeFromString(String storageType) { - if (storageType == null) return NodeResources.StorageType.getDefault(); - switch (storageType) { - case "remote": return NodeResources.StorageType.remote; - case "local": return NodeResources.StorageType.local; - case "any": return NodeResources.StorageType.any; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); - } - } - - private static Node.ServiceState toServiceState(OrchestratorStatus orchestratorStatus) { - switch (orchestratorStatus) { - case ALLOWED_TO_BE_DOWN: return Node.ServiceState.allowedDown; - case PERMANENTLY_DOWN: return Node.ServiceState.permanentlyDown; - case NO_REMARKS: return Node.ServiceState.expectedUp; - } - - return Node.ServiceState.unknown; - } - - private static double toDouble(Double d) { - return d == null ? 0 : d; - } - - private static int toInt(Integer i) { - return i == null ? 0 : i; - } - - private static Version versionFrom(String s) { - return s == null ? Version.emptyVersion : Version.fromString(s); - } - - private static DockerImage dockerImageFrom(String s) { - return s == null ? DockerImage.EMPTY : DockerImage.fromString(s); - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java index c2425fe0f72..97c6222e77d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/repair/RepairTicketReport.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.repair; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; @@ -53,8 +52,8 @@ public class RepairTicketReport { return REPORT_ID; } - public static RepairTicketReport fromJsonNode(JsonNode node) { - return uncheck(() -> objectMapper.treeToValue(node, RepairTicketReport.class)); + public static RepairTicketReport fromJsonNode(String jsonReport) { + return uncheck(() -> objectMapper.readValue(jsonReport, RepairTicketReport.class)); } public JsonNode toJsonNode() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java index a3c0af95053..8ebfde9f475 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VCMRReport.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.vcmr; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; @@ -69,12 +68,12 @@ public class VCMRReport { /** * Serialization functions - mapped to {@link Node#reports()} */ - public static VCMRReport fromReports(Map<String, JsonNode> reports) { + public static VCMRReport fromReports(Map<String, String> reports) { var serialized = reports.get(REPORT_ID); if (serialized == null) return new VCMRReport(); - return uncheck(() -> objectMapper.treeToValue(serialized, VCMRReport.class)); + return uncheck(() -> objectMapper.readValue(serialized, VCMRReport.class)); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index f8c390edb3b..157cdea831e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -28,7 +28,11 @@ import java.util.stream.Collectors; */ public class NameServiceForwarder { - private static final int QUEUE_CAPACITY = 300; + /** + * The number of {@link NameServiceRequest}s we allow to be queued. When the queue overflows, the first requests + * are dropped in a FIFO order until the queue shrinks below this capacity. + */ + private static final int QUEUE_CAPACITY = 400; private static final Logger log = Logger.getLogger(NameServiceForwarder.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index d74578d9adc..fc7b256a644 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -2,13 +2,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import java.util.Collection; import java.util.List; @@ -16,6 +13,9 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +/** + * @author smorgrav + */ public class ChangeManagementAssessor { private final NodeRepository nodeRepository; @@ -25,31 +25,31 @@ public class ChangeManagementAssessor { } public Assessment assessment(List<String> impactedHostnames, ZoneId zone) { - return assessmentInner(impactedHostnames, nodeRepository.listNodes(zone).nodes(), zone); + return assessmentInner(impactedHostnames, nodeRepository.list(zone, false), zone); } - Assessment assessmentInner(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes, ZoneId zone) { + Assessment assessmentInner(List<String> impactedHostnames, List<Node> allNodes, ZoneId zone) { List<String> impactedParentHosts = toParentHosts(impactedHostnames, allNodes); // Group impacted application nodes by parent host - Map<NodeRepositoryNode, List<NodeRepositoryNode>> prParentHost = allNodes.stream() - .filter(nodeRepositoryNode -> nodeRepositoryNode.getState() == NodeState.active) //TODO look at more states? - .filter(node -> impactedParentHosts.contains(node.getParentHostname() == null ? "" : node.getParentHostname())) + Map<Node, List<Node>> prParentHost = allNodes.stream() + .filter(node -> node.state() == Node.State.active) //TODO look at more states? + .filter(node -> impactedParentHosts.contains(node.parentHostname().map(HostName::value).orElse(""))) .collect(Collectors.groupingBy(node -> allNodes.stream() - .filter(parent -> parent.getHostname().equals(node.getParentHostname())) + .filter(parent -> parent.hostname().equals(node.parentHostname().get())) .findFirst().orElseThrow() )); // Group nodes pr cluster - Map<Cluster, List<NodeRepositoryNode>> prCluster = prParentHost.values() + Map<Cluster, List<Node>> prCluster = prParentHost.values() .stream() .flatMap(Collection::stream) .collect(Collectors.groupingBy(ChangeManagementAssessor::clusterKey)); var tenantHosts = prParentHost.keySet().stream() - .filter(node -> node.getType() == NodeType.host) - .map(node -> HostName.from(node.getHostname())) + .filter(node -> node.type() == NodeType.host) + .map(node -> node.hostname()) .collect(Collectors.toList()); boolean allHostsReplacable = tenantHosts.isEmpty() || nodeRepository.isReplaceable( @@ -60,7 +60,7 @@ public class ChangeManagementAssessor { // Report assessment pr cluster var clusterAssessments = prCluster.entrySet().stream().map((entry) -> { Cluster cluster = entry.getKey(); - List<NodeRepositoryNode> nodes = entry.getValue(); + List<Node> nodes = entry.getValue(); long[] totalStats = clusterStats(cluster, allNodes); long[] impactedStats = clusterStats(cluster, nodes); @@ -87,8 +87,8 @@ public class ChangeManagementAssessor { var hostAssessments = prParentHost.entrySet().stream().map((entry) -> { HostAssessment hostAssessment = new HostAssessment(); - hostAssessment.hostName = entry.getKey().getHostname(); - hostAssessment.switchName = entry.getKey().getSwitchHostname(); + hostAssessment.hostName = entry.getKey().hostname().value(); + hostAssessment.switchName = entry.getKey().switchHostname().orElse(null); hostAssessment.numberOfChildren = entry.getValue().size(); //TODO: Some better heuristic for what's considered problematic @@ -103,31 +103,31 @@ public class ChangeManagementAssessor { return new Assessment(clusterAssessments, hostAssessments); } - private List<String> toParentHosts(List<String> impactedHostnames, List<NodeRepositoryNode> allNodes) { + private List<String> toParentHosts(List<String> impactedHostnames, List<Node> allNodes) { return impactedHostnames.stream() .flatMap(hostname -> allNodes.stream() - .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.getType())) - .filter(node -> hostname.equals(node.getHostname()) || hostname.equals(node.getParentHostname())) + .filter(node -> List.of(NodeType.config, NodeType.proxy, NodeType.host).contains(node.type())) + .filter(node -> hostname.equals(node.hostname().value()) || hostname.equals(node.parentHostname().map(HostName::value).orElse(""))) .map(node -> { - if (node.getType() == NodeType.host) - return node.getHostname(); - return node.getParentHostname(); + if (node.type() == NodeType.host) + return node.hostname().value(); + return node.parentHostname().get().value(); }).findFirst().stream() ) .collect(Collectors.toList()); } - private static Cluster clusterKey(NodeRepositoryNode node) { - if (node.getOwner() == null) + private static Cluster clusterKey(Node node) { + if (node.owner().isEmpty()) return Cluster.EMPTY; - String appId = Text.format("%s:%s:%s", node.getOwner().tenant, node.getOwner().application, node.getOwner().instance); - return new Cluster(Node.ClusterType.valueOf(node.getMembership().clustertype), node.getMembership().clusterid, appId, node.getType()); + String appId = node.owner().get().serializedForm(); + return new Cluster(node.clusterType(), node.clusterId(), appId, node.type()); } - private static long[] clusterStats(Cluster cluster, List<NodeRepositoryNode> containerNodes) { - List<NodeRepositoryNode> clusterNodes = containerNodes.stream().filter(nodeRepositoryNode -> cluster.equals(clusterKey(nodeRepositoryNode))).collect(Collectors.toList()); - long groups = clusterNodes.stream().map(nodeRepositoryNode -> nodeRepositoryNode.getMembership() != null ? nodeRepositoryNode.getMembership().group : "").distinct().count(); + private static long[] clusterStats(Cluster cluster, List<Node> containerNodes) { + List<Node> clusterNodes = containerNodes.stream().filter(node -> cluster.equals(clusterKey(node))).collect(Collectors.toList()); + long groups = clusterNodes.stream().map(Node::group).distinct().count(); return new long[] { clusterNodes.size(), groups}; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java index 38e10aa6750..19617a1f293 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainer.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; @@ -174,7 +175,7 @@ public class VCMRMaintainer extends ControllerMaintainer { } catch (Exception e) { logger.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); // Check if retirement actually failed - if (!nodeRepository.getNode(changeRequest.getZoneId(), node.hostname().value()).getWantToRetire()) { + if (!nodeRepository.getNode(changeRequest.getZoneId(), node.hostname().value()).wantToRetire()) { return hostAction; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index c1e6c362c83..1a4fa3fcd79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -880,7 +880,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.getEvent()))); + nodeObject.setBool("down", node.history().stream().anyMatch(event -> "down".equals(event.name()))); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index bfcefecba0c..58afed4143f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -11,7 +11,6 @@ import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -26,7 +25,6 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import javax.ws.rs.BadRequestException; -import java.io.IOException; import java.math.BigDecimal; import java.time.Clock; import java.time.Instant; @@ -34,8 +32,8 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Comparator; -import java.util.Optional; import java.util.List; +import java.util.Optional; /** * @author ogronnesby @@ -78,11 +76,6 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) - /* - * Utility - map Slime.class => SlimeJsonResponse - */ - .addRequestMapper(Slime.class, BillingApiHandlerV2::slimeRequestMapper) - .addResponseMapper(Slime.class, BillingApiHandlerV2::slimeResponseMapper) .build(); } @@ -337,16 +330,4 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return inspector.field(field).asString(); } - private static Optional<Slime> slimeRequestMapper(RestApi.RequestContext requestContext) { - try { - return Optional.of(SlimeUtils.jsonToSlime(requestContext.requestContentOrThrow().content().readAllBytes())); - } catch (IOException e) { - throw new IllegalArgumentException("Could not parse JSON input"); - } - } - - private static HttpResponse slimeResponseMapper(RestApi.RequestContext ctx, Slime slime) { - return new SlimeJsonResponse(slime); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index d8bddac4187..5e79f1e4d12 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -197,10 +197,10 @@ public class InternalStepRunnerTest { app.instanceId()).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), - new Node.Builder(systemTestNode) - .serviceState(Node.ServiceState.allowedDown) - .suspendedSince(tester.clock().instant()) - .build()); + Node.builder(systemTestNode) + .serviceState(Node.ServiceState.allowedDown) + .suspendedSince(tester.clock().instant()) + .build()); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 92c8cbc4889..c66cc3710c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -63,10 +63,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BooleanSupplier; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; @@ -138,23 +135,23 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer Node parent = nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream().findAny() .orElseThrow(() -> new IllegalStateException("No parent hosts in " + zone)); - nodeRepository().putNodes(zone, new Node.Builder().hostname(hostFor(application, zone)) - .state(Node.State.reserved) - .type(NodeType.tenant) - .owner(application) - .parentHostname(parent.hostname()) - .currentVersion(initialVersion) - .wantedVersion(initialVersion) - .currentDockerImage(initialDockerImage) - .wantedDockerImage(initialDockerImage) - .currentOsVersion(Version.emptyVersion) - .wantedOsVersion(Version.emptyVersion) - .resources(new NodeResources(2, 8, 50, 1, slow, remote)) - .serviceState(Node.ServiceState.unorchestrated) - .flavor("d-2-8-50") - .clusterId(clusterId.value()) - .clusterType(Node.ClusterType.container) - .build()); + nodeRepository().putNodes(zone, Node.builder().hostname(hostFor(application, zone)) + .state(Node.State.reserved) + .type(NodeType.tenant) + .owner(application) + .parentHostname(parent.hostname()) + .currentVersion(initialVersion) + .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) + .currentOsVersion(Version.emptyVersion) + .wantedOsVersion(Version.emptyVersion) + .resources(new NodeResources(2, 8, 50, 1, slow, remote)) + .serviceState(Node.ServiceState.unorchestrated) + .flavor("d-2-8-50") + .clusterId(clusterId.value()) + .clusterType(Node.ClusterType.container) + .build()); } public HostName hostFor(ApplicationId application, ZoneId zone) { @@ -174,16 +171,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (ZoneId zone : zones) { for (SystemApplication application : applications) { for (int i = 1; i <= 3; i++) { - Node node = new Node.Builder() - .hostname(HostName.from("node-" + i + "-" + application.id().application() + Node node = Node.builder() + .hostname(HostName.from("node-" + i + "-" + application.id().application() .value() + "-" + zone.value())) - .state(Node.State.active) - .type(application.nodeType()) - .owner(application.id()) - .currentVersion(initialVersion).wantedVersion(initialVersion) - .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) - .build(); - nodeRepository().putNode(zone, node); + .state(Node.State.active) + .type(application.nodeType()) + .owner(application.id()) + .currentVersion(initialVersion).wantedVersion(initialVersion) + .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) + .build(); + nodeRepository().putNodes(zone, node); } convergeServices(application.id(), zone); } @@ -244,9 +241,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (var node : nodes) { Node newNode; if (osVersion) { - newNode = new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build(); + newNode = Node.builder(node).currentOsVersion(version).wantedOsVersion(version).build(); } else { - newNode = new Node.Builder(node).currentVersion(version).wantedVersion(version).build(); + newNode = Node.builder(node).currentVersion(version).wantedVersion(version).build(); } nodeRepository().putNodes(zone, newNode); } @@ -410,10 +407,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer application.activate(); List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId()); for (Node node : nodes) { - nodeRepository.putNodes(id.zoneId(), new Node.Builder(node) - .state(Node.State.active) - .wantedVersion(application.version().get()) - .build()); + nodeRepository.putNodes(id.zoneId(), Node.builder(node) + .state(Node.State.active) + .wantedVersion(application.version().get()) + .build()); } serviceStatus.put(id, new ServiceConvergence(id.applicationId(), id.zoneId(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 4079591730d..b16817c0f3d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -18,13 +17,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepoStats; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVersions; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import java.net.URI; import java.time.Duration; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -33,7 +30,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -50,115 +46,12 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new HashMap<>(); private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new HashMap<>(); - // A separate/alternative list of NodeRepositoryNode nodes. - // Methods operating with Node and NodeRepositoryNode lives separate lives. - private final Map<ZoneId, List<NodeRepositoryNode>> nodeRepoNodes = new HashMap<>(); - private boolean allowPatching = false; private boolean hasSpareCapacity = false; - /** Add or update given nodes in zone */ - public void putNodes(ZoneId zone, List<Node> nodes) { - Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); - for (var node : nodes) { - zoneNodes.put(node.hostname(), node); - } - } - - public void putNode(ZoneId zone, Node node) { - nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()).put(node.hostname(), node); - } - - public void putApplication(ZoneId zone, Application application) { - applications.putIfAbsent(zone, new HashMap<>()); - applications.get(zone).put(application.id(), application); - } - - @Override - public NodeRepoStats getStats(ZoneId zone) { - List<ApplicationStats> applicationStats = - applications.containsKey(zone) - ? applications.get(zone).keySet().stream() - .map(id -> new ApplicationStats(id, Load.zero(), 0, 0)) - .collect(Collectors.toList()) - : List.of(); - - return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); - } - - public Pair<Double, Double> getTrafficFraction(ApplicationId application, ZoneId zone) { - return trafficFractions.get(new DeploymentId(application, zone)); - } - - /** Add or update given node in zone */ - public void putNodes(ZoneId zone, Node node) { - putNodes(zone, Collections.singletonList(node)); - } - - /** Remove given nodes from zone */ - public void removeNodes(ZoneId zone, List<Node> nodes) { - nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); - } - - /** Remove all nodes in all zones */ - public void clear() { - nodeRepository.clear(); - nodeRepoNodes.clear(); - } - - /** Replace nodes in zone with given nodes */ - public void setNodes(ZoneId zone, List<Node> nodes) { - nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); - } - - public Node require(HostName hostName) { - return nodeRepository.values().stream() - .map(zoneNodes -> zoneNodes.get(hostName)) - .filter(Objects::nonNull) - .findFirst() - .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); - } - - /** Replace nodes in zone with a fixed set of nodes */ - public void setFixedNodes(ZoneId zone) { - var nodeA = new Node.Builder() - .hostname(HostName.from("hostA")) - .parentHostname(HostName.from("parentHostA")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(Node.ClusterType.container) - .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) - .build(); - var nodeB = new Node.Builder() - .hostname(HostName.from("hostB")) - .parentHostname(HostName.from("parentHostB")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant2", "app2", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(40, 24, 500, 1)) - .cost(20) - .clusterId("clusterB") - .clusterType(Node.ClusterType.container) - .build(); - setNodes(zone, List.of(nodeA, nodeB)); - } - @Override public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) { - nodeRepoNodes.put(zone, new ArrayList<>(nodes)); + throw new UnsupportedOperationException(); } @Override @@ -171,23 +64,18 @@ public class NodeRepositoryMock implements NodeRepository { var existing = list(zone, List.of(HostName.from(hostName))); if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zone); - var node = new Node.Builder(existing.get(0)) - .state(Node.State.valueOf(nodeState.name())) - .build(); + var node = Node.builder(existing.get(0)) + .state(Node.State.valueOf(nodeState.name())) + .build(); putNodes(zone, node); } @Override - public NodeRepositoryNode getNode(ZoneId zone, String hostname) { + public Node getNode(ZoneId zone, String hostname) { throw new UnsupportedOperationException(); } @Override - public NodeList listNodes(ZoneId zone) { - return new NodeList(nodeRepoNodes.get(zone)); - } - - @Override public List<Node> list(ZoneId zone, boolean includeDeprovisioned) { return List.copyOf(nodeRepository.getOrDefault(zone, Map.of()).values()); } @@ -218,6 +106,18 @@ public class NodeRepositoryMock implements NodeRepository { } @Override + public NodeRepoStats getStats(ZoneId zone) { + List<ApplicationStats> applicationStats = + applications.containsKey(zone) + ? applications.get(zone).keySet().stream() + .map(id -> new ApplicationStats(id, Load.zero(), 0, 0)) + .collect(Collectors.toList()) + : List.of(); + + return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); + } + + @Override public Map<TenantName, URI> getArchiveUris(ZoneId zone) { return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); } @@ -244,7 +144,7 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) - .map(node -> new Node.Builder(node).wantedVersion(version).build()) + .map(node -> Node.builder(node).wantedVersion(version).build()) .forEach(node -> putNodes(zone, node)); } @@ -261,7 +161,7 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Map.of()).values() .stream() .filter(node -> node.type() == type) - .map(node -> new Node.Builder(node).wantedOsVersion(version).build()) + .map(node -> Node.builder(node).wantedOsVersion(version).build()) .forEach(node -> putNodes(zone, node)); } @@ -290,15 +190,20 @@ public class NodeRepositoryMock implements NodeRepository { if (existing.size() != 1) throw new IllegalArgumentException("Node " + hostName + " not found in " + zoneId); // Note: Only supports switchHostname, modelName and wantToRetire - Node.Builder newNode = new Node.Builder(existing.get(0)); + Node.Builder newNode = Node.builder(existing.get(0)); if (node.getSwitchHostname() != null) newNode.switchHostname(node.getSwitchHostname()); if (node.getModelName() != null) newNode.modelName(node.getModelName()); if (node.getWantToRetire() != null) newNode.wantToRetire(node.getWantToRetire()); - if (!node.getReports().isEmpty()) - newNode.reports(node.getReports()); + + Map<String, String> reports = new HashMap<>(); + for (var kv : node.getReports().entrySet()) { + if (kv.getValue() == null) continue; // Null value clears a report + reports.put(kv.getKey(), kv.getValue().toString()); + } + newNode.reports(reports); putNodes(zoneId, newNode.build()); } @@ -313,6 +218,83 @@ public class NodeRepositoryMock implements NodeRepository { return hasSpareCapacity; } + /** Add or update given nodes in zone */ + public void putNodes(ZoneId zone, List<Node> nodes) { + Map<HostName, Node> zoneNodes = nodeRepository.computeIfAbsent(zone, (k) -> new HashMap<>()); + for (var node : nodes) { + zoneNodes.put(node.hostname(), node); + } + } + + /** Add or update given node in zone */ + public void putNodes(ZoneId zone, Node node) { + putNodes(zone, List.of(node)); + } + + public void putApplication(ZoneId zone, Application application) { + applications.computeIfAbsent(zone, (k) -> new HashMap<>()) + .put(application.id(), application); + } + + public Pair<Double, Double> getTrafficFraction(ApplicationId application, ZoneId zone) { + return trafficFractions.get(new DeploymentId(application, zone)); + } + + /** Remove given nodes from zone */ + public void removeNodes(ZoneId zone, List<Node> nodes) { + nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); + } + + /** Remove all nodes in all zones */ + public void clear() { + nodeRepository.clear(); + } + + public Node require(HostName hostName) { + return nodeRepository.values().stream() + .map(zoneNodes -> zoneNodes.get(hostName)) + .filter(Objects::nonNull) + .findFirst() + .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); + } + + /** Add a fixed set of nodes to given zone */ + public void addFixedNodes(ZoneId zone) { + var nodeA = Node.builder() + .hostname(HostName.from("hostA")) + .parentHostname(HostName.from("parentHostA")) + .state(Node.State.active) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(Node.ClusterType.container) + .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) + .build(); + var nodeB = Node.builder() + .hostname(HostName.from("hostB")) + .parentHostname(HostName.from("parentHostB")) + .state(Node.State.active) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant2", "app2", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(40, 24, 500, 1)) + .cost(20) + .clusterId("clusterB") + .clusterType(Node.ClusterType.container) + .build(); + putNodes(zone, List.of(nodeA, nodeB)); + } + public Optional<Duration> osUpgradeBudget(ZoneId zone, NodeType type, Version version) { return Optional.ofNullable(osUpgradeBudgets.get(Objects.hash(zone, type, version))); } @@ -320,10 +302,10 @@ public class NodeRepositoryMock implements NodeRepository { public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); - return new Node.Builder(node) - .currentVersion(version) - .currentDockerImage(node.wantedDockerImage()) - .build(); + return Node.builder(node) + .currentVersion(version) + .currentDockerImage(node.wantedDockerImage()) + .build(); }); } @@ -336,23 +318,29 @@ public class NodeRepositoryMock implements NodeRepository { } public void requestRestart(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).wantedRestartGeneration(node.wantedRestartGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> Node.builder(node).wantedRestartGeneration(node.wantedRestartGeneration() + 1).build()); } public void doRestart(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).restartGeneration(node.restartGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> Node.builder(node).restartGeneration(node.restartGeneration() + 1).build()); } public void requestReboot(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).wantedRebootGeneration(node.wantedRebootGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> Node.builder(node).wantedRebootGeneration(node.wantedRebootGeneration() + 1).build()); } public void doReboot(DeploymentId deployment, Optional<HostName> hostname) { - modifyNodes(deployment, hostname, node -> new Node.Builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); + modifyNodes(deployment, hostname, node -> Node.builder(node).rebootGeneration(node.rebootGeneration() + 1).build()); } - public void addReport(ZoneId zoneId, HostName hostName, String reportId, JsonNode report) { - nodeRepository.get(zoneId).get(hostName).reports().put(reportId, report); + public void addReport(ZoneId zoneId, HostName hostName, String reportId, String report) { + Node node = nodeRepository.getOrDefault(zoneId, Map.of()).get(hostName); + if (node == null) throw new IllegalArgumentException("No node named " + hostName + " in " + zoneId); + + Map<String, String> reports = new HashMap<>(node.reports()); + reports.put(reportId, report); + Node newNode = Node.builder(node).reports(reports).build(); + putNodes(zoneId, newNode); } public NodeRepositoryMock allowPatching(boolean allowPatching) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java index 476d2465202..01c16139667 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessorTest.java @@ -2,32 +2,32 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * @author smorgrav + */ public class ChangeManagementAssessorTest { - private ChangeManagementAssessor changeManagementAssessor = new ChangeManagementAssessor(new NodeRepositoryMock()); + private final ChangeManagementAssessor changeManagementAssessor = new ChangeManagementAssessor(new NodeRepositoryMock()); @Test public void empty_input_variations() { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = new ArrayList<>(); - List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); + List<Node> allNodesInZone = new ArrayList<>(); // Both zone and hostnames are empty ChangeManagementAssessor.Assessment assessment @@ -39,7 +39,7 @@ public class ChangeManagementAssessorTest { public void one_host_one_cluster_no_groups() { ZoneId zone = ZoneId.from("prod", "eu-trd"); List<String> hostNames = Collections.singletonList("host1"); - List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); + List<Node> allNodesInZone = new ArrayList<>(); allNodesInZone.add(createNode("node1", "host1", "default", 0 )); allNodesInZone.add(createNode("node2", "host1", "default", 0 )); allNodesInZone.add(createNode("node3", "host1", "default", 0 )); @@ -69,8 +69,8 @@ public class ChangeManagementAssessorTest { @Test public void one_of_two_groups_in_one_of_two_clusters() { ZoneId zone = ZoneId.from("prod", "eu-trd"); - List<String> hostNames = Arrays.asList("host1", "host2", "host5"); - List<NodeRepositoryNode> allNodesInZone = new ArrayList<>(); + List<String> hostNames = List.of("host1", "host2", "host5"); + List<Node> allNodesInZone = new ArrayList<>(); // Two impacted nodes on host1 allNodesInZone.add(createNode("node1", "host1", "default", 0 )); @@ -123,8 +123,8 @@ public class ChangeManagementAssessorTest { @Test public void two_config_nodes() { var zone = ZoneId.from("prod", "eu-trd"); - var hostNames = Arrays.asList("config1", "config2"); - var allNodesInZone = new ArrayList<NodeRepositoryNode>(); + var hostNames = List.of("config1", "config2"); + var allNodesInZone = new ArrayList<Node>(); // Add config nodes and parents allNodesInZone.add(createNode("config1", "confighost1", "config", 0, NodeType.config)); @@ -141,8 +141,8 @@ public class ChangeManagementAssessorTest { @Test public void one_of_three_proxy_nodes() { var zone = ZoneId.from("prod", "eu-trd"); - var hostNames = Arrays.asList("routing1"); - var allNodesInZone = new ArrayList<NodeRepositoryNode>(); + var hostNames = List.of("routing1"); + var allNodesInZone = new ArrayList<Node>(); // Add routing nodes and parents allNodesInZone.add(createNode("routing1", "parentrouting1", "routing", 0, NodeType.proxy)); @@ -156,47 +156,33 @@ public class ChangeManagementAssessorTest { assertEquals("33% of routing nodes impacted. Consider reprovisioning if too many", assessment.get(0).impact); } - private NodeOwner createOwner() { - NodeOwner owner = new NodeOwner(); - owner.tenant = "mytenant"; - owner.application = "myapp"; - owner.instance = "default"; - return owner; - } - - private NodeMembership createMembership(String clusterId, int group) { - NodeMembership membership = new NodeMembership(); - membership.group = "" + group; - membership.clusterid = clusterId; - membership.clustertype = "content"; - membership.index = 2; - membership.retired = false; - return membership; - } - - private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { + private Node createNode(String nodename, String hostname, String clusterId, int group) { return createNode(nodename, hostname, clusterId, group, NodeType.tenant); } - private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(nodename); - node.setParentHostname(hostname); - node.setState(NodeState.active); - node.setOwner(createOwner()); - node.setMembership(createMembership(clusterId, group)); - node.setType(nodeType); - - return node; + private Node createNode(String nodename, String hostname, String clusterId, int group, NodeType nodeType) { + return Node.builder().hostname(nodename) + .parentHostname(hostname) + .state(Node.State.active) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .group(String.valueOf(group)) + .clusterId(clusterId) + .clusterType(Node.ClusterType.content) + .type(nodeType) + .build(); } - private NodeRepositoryNode createHost(String hostname, NodeType nodeType) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(hostname); - node.setSwitchHostname("switch1"); - node.setType(nodeType); - node.setOwner(createOwner()); - node.setMembership(createMembership(nodeType.name(), 0)); - return node; + private Node createHost(String hostname, NodeType nodeType) { + return Node.builder() + .hostname(hostname) + .switchHostname("switch1") + .state(Node.State.active) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .group(String.valueOf(0)) + .clusterId(nodeType.name()) + .clusterType(Node.ClusterType.content) + .type(nodeType) + .build(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index 680743055c9..e838a693be1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -121,10 +121,10 @@ public class CloudEventReporterTest { } private Node createNode(String hostname, NodeType nodeType) { - return new Node.Builder() - .hostname(HostName.from(hostname)) - .type(nodeType) - .build(); + return Node.builder() + .hostname(HostName.from(hostname)) + .type(nodeType) + .build(); } private Set<String> getHostnames(ZoneId zoneId) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index cbffd6d610f..d78b48c362a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -54,7 +54,7 @@ public class CostReportMaintainerTest { private void addNodes() { for (var zone : tester.zoneRegistry().zones().all().zones()) { - tester.configServer().nodeRepository().setFixedNodes(zone.getId()); + tester.configServer().nodeRepository().addFixedNodes(zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java index 0baee28143c..56f8de494f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdaterTest.java @@ -83,12 +83,12 @@ public class HostInfoUpdaterTest { // Updates node registered under a different hostname ZoneId zone = tester.zoneRegistry().zones().controllerUpgraded().all().ids().get(0); String hostnameSuffix = ".prod." + zone.value(); - Node configNode = new Node.Builder().hostname(HostName.from("cfg3" + hostnameSuffix)) - .type(NodeType.config) - .build(); - Node configHost = new Node.Builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) - .type(NodeType.confighost) - .build(); + Node configNode = Node.builder().hostname(HostName.from("cfg3" + hostnameSuffix)) + .type(NodeType.config) + .build(); + Node configHost = Node.builder().hostname(HostName.from("cfghost3" + hostnameSuffix)) + .type(NodeType.confighost) + .build(); tester.serviceRegistry().configServer().nodeRepository().putNodes(zone, List.of(configNode, configHost)); String switchHostname = switchHostname(configHost); NodeEntity configNodeEntity = new NodeEntity("cfg3" + hostnameSuffix, "RD350G", "Lenovo", switchHostname); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 29c5573a1f5..3789777a8fc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -28,7 +28,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -546,7 +545,7 @@ public class MetricsReporterTest { ControllerTester tester) { var currentNodes = getNodes(zone, nodes, tester); var updatedNodes = currentNodes.stream() - .map(node -> builderOps.apply(new Node.Builder(node)).build()) + .map(node -> builderOps.apply(Node.builder(node)).build()) .collect(Collectors.toList()); tester.configServer().nodeRepository().putNodes(zone, updatedNodes); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index c29e10ab643..eca000ff969 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -273,7 +273,7 @@ public class OsUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putNodes(zone.getVirtualId(), new Node.Builder(node).state(Node.State.failed).build()); + nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).state(Node.State.failed).build()); } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ @@ -285,9 +285,9 @@ public class OsUpgraderTest { assertWanted(wantedVersion, application, zones); for (ZoneApi zone : zones) { for (Node node : nodesRequiredToUpgrade(zone, application)) { - nodeRepository().putNodes(zone.getVirtualId(), new Node.Builder(node).wantedOsVersion(version) - .currentOsVersion(version) - .build()); + nodeRepository().putNodes(zone.getVirtualId(), Node.builder(node).wantedOsVersion(version) + .currentOsVersion(version) + .build()); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index e61516cbb1a..5ef64b460b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -114,8 +114,8 @@ public class ResourceMeterMaintainerTest { ZoneApiMock zone1 = ZoneApiMock.newBuilder().withId("prod.region-2").build(); ZoneApiMock zone2 = ZoneApiMock.newBuilder().withId("test.region-3").build(); tester.zoneRegistry().setZones(zone1, zone2); - tester.configServer().nodeRepository().setFixedNodes(zone1.getId()); - tester.configServer().nodeRepository().setFixedNodes(zone2.getId()); + tester.configServer().nodeRepository().addFixedNodes(zone1.getId()); + tester.configServer().nodeRepository().addFixedNodes(zone2.getId()); tester.configServer().nodeRepository().putNodes(zone1.getId(), createNodes()); } @@ -126,21 +126,21 @@ public class ResourceMeterMaintainerTest { Node.State.failed, Node.State.parked, Node.State.active) - .map(state -> new Node.Builder() - .hostname(HostName.from("host" + state)) - .parentHostname(HostName.from("parenthost" + state)) - .state(state) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(24, 24, 500, 1)) - .clusterId("clusterA") - .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) - .build()) + .map(state -> Node.builder() + .hostname(HostName.from("host" + state)) + .parentHostname(HostName.from("parenthost" + state)) + .state(state) + .type(NodeType.tenant) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .currentVersion(Version.fromString("7.42")) + .wantedVersion(Version.fromString("7.42")) + .currentOsVersion(Version.fromString("7.6")) + .wantedOsVersion(Version.fromString("7.6")) + .serviceState(Node.ServiceState.expectedUp) + .resources(new NodeResources(24, 24, 500, 1)) + .clusterId("clusterA") + .clusterType(state == Node.State.active ? Node.ClusterType.admin : Node.ClusterType.container) + .build()) .collect(Collectors.toUnmodifiableList()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 516c28ab5cd..a90c8a9593b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -15,7 +15,6 @@ import org.junit.Test; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.Optional; import static com.yahoo.vespa.hosted.controller.maintenance.ResourceTagMaintainer.SHARED_HOST_APPLICATION; import static org.junit.Assert.assertEquals; @@ -25,7 +24,7 @@ import static org.junit.Assert.assertEquals; */ public class ResourceTagMaintainerTest { - final ControllerTester tester = new ControllerTester(); + private final ControllerTester tester = new ControllerTester(); @Test public void maintain() { @@ -51,24 +50,24 @@ public class ResourceTagMaintainerTest { } public void setNodes(ZoneId zone) { - var hostA = new Node.Builder() - .hostname(HostName.from("parentHostA." + zone.value())) - .type(NodeType.host) - .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) - .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) - .build(); - var nodeA = new Node.Builder() - .hostname(HostName.from("hostA." + zone.value())) - .type(NodeType.tenant) - .parentHostname(HostName.from("parentHostA." + zone.value())) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .build(); - var hostB = new Node.Builder() - .hostname(HostName.from("parentHostB." + zone.value())) - .type(NodeType.host) - .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) - .build(); - tester.configServer().nodeRepository().setNodes(zone, List.of(hostA, nodeA, hostB)); + var hostA = Node.builder() + .hostname(HostName.from("parentHostA." + zone.value())) + .type(NodeType.host) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) + .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) + .build(); + var nodeA = Node.builder() + .hostname(HostName.from("hostA." + zone.value())) + .type(NodeType.tenant) + .parentHostname(HostName.from("parentHostA." + zone.value())) + .owner(ApplicationId.from("tenant1", "app1", "default")) + .build(); + var hostB = Node.builder() + .hostname(HostName.from("parentHostB." + zone.value())) + .type(NodeType.host) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) + .build(); + tester.configServer().nodeRepository().putNodes(zone, List.of(hostA, nodeA, hostB)); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index db2353860ae..8090765b5f9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -305,7 +305,7 @@ public class SystemUpgraderTest { for (Node node : listNodes(zone, application)) { nodeRepository().putNodes( zone.getId(), - new Node.Builder(node).currentVersion(node.wantedVersion()).build()); + Node.builder(node).currentVersion(node.wantedVersion()).build()); } assertCurrentVersion(application, version, zone); }); @@ -329,7 +329,7 @@ public class SystemUpgraderTest { Node node = nodes.get(0); nodeRepository().putNodes( zone.getId(), - new Node.Builder(node).state(Node.State.failed).build()); + Node.builder(node).state(Node.State.failed).build()); } private void assertSystemVersion(Version version) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java index 16ed6b7ef98..f957b14ef95 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VCMRMaintainerTest.java @@ -20,8 +20,12 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author olaa @@ -49,9 +53,12 @@ public class VCMRMaintainerTest { vcmrReport.addVcmr("id123", ZonedDateTime.now(), ZonedDateTime.now()); var parkedNode = createNode(host1, NodeType.host, Node.State.parked, true); var failedNode = createNode(host2, NodeType.host, Node.State.failed, false); - parkedNode = new Node.Builder(parkedNode) - .reports(vcmrReport.toNodeReports()) - .build(); + Map<String, String> reports = vcmrReport.toNodeReports().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + kv -> kv.getValue().toString())); + parkedNode = Node.builder(parkedNode) + .reports(reports) + .build(); nodeRepo.putNodes(zoneId, List.of(parkedNode, failedNode)); @@ -63,8 +70,7 @@ public class VCMRMaintainerTest { assertEquals(Node.State.dirty, nodeList.get(0).state()); assertEquals(Node.State.failed, nodeList.get(1).state()); - var report = nodeList.get(0).reports(); - assertNull(report.get(VCMRReport.getReportId())); + assertTrue(nodeList.get(0).reports().isEmpty()); var writtenChangeRequest = tester.curator().readChangeRequest(changeRequestId).get(); assertEquals(Status.COMPLETED, writtenChangeRequest.getStatus()); @@ -235,11 +241,11 @@ public class VCMRMaintainerTest { } private Node createNode(HostName hostname, NodeType nodeType, Node.State state, boolean wantToRetire) { - return new Node.Builder() - .hostname(hostname) - .type(nodeType) - .state(state) - .wantToRetire(wantToRetire) - .build(); + return Node.builder() + .hostname(hostname) + .type(nodeType) + .state(state) + .wantToRetire(wantToRetire) + .build(); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 80cee3af58b..5846ab5f2a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -2,23 +2,17 @@ package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.application.container.handler.Request; -import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequest; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestSource; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.HostAction; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import org.intellij.lang.annotations.Language; import org.junit.Before; import org.junit.Test; @@ -42,8 +36,7 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { public void before() { tester = new ContainerTester(container, responses); addUserToHostedOperatorRole(operator); - tester.serviceRegistry().configServer().nodeRepository().addNodes(ZoneId.from("prod.us-east-3"), createNodes()); - tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNode()); + tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNodes()); tester.controller().curator().writeChangeRequest(createChangeRequest()); } @@ -85,23 +78,11 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { assertEquals(VespaChangeRequest.Status.COMPLETED, changeRequest.getStatus()); } - private void assertResponse(Request request, @Language("JSON") String body, int statusCode) { - addIdentityToRequest(request, operator); - tester.assertResponse(request, body, statusCode); - } - private void assertFile(Request request, String filename) { addIdentityToRequest(request, operator); tester.assertResponse(request, new File(filename)); } - private Node createNode() { - return new Node.Builder() - .hostname(HostName.from("host1")) - .switchHostname("switch1") - .build(); - } - private VespaChangeRequest createChangeRequest() { var instant = Instant.ofEpochMilli(9001); var date = ZonedDateTime.ofInstant(instant, java.time.ZoneId.of("UTC")); @@ -124,8 +105,8 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { ); } - private List<NodeRepositoryNode> createNodes() { - List<NodeRepositoryNode> nodes = new ArrayList<>(); + private List<Node> createNodes() { + List<Node> nodes = new ArrayList<>(); nodes.add(createNode("node1", "host1", "default", 0 )); nodes.add(createNode("node2", "host1", "default", 0 )); nodes.add(createNode("node3", "host1", "default", 0 )); @@ -135,44 +116,27 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { return nodes; } - private NodeOwner createOwner() { - NodeOwner owner = new NodeOwner(); - owner.tenant = "mytenant"; - owner.application = "myapp"; - owner.instance = "default"; - return owner; - } - - private NodeMembership createMembership(String clusterId, int group) { - NodeMembership membership = new NodeMembership(); - membership.group = "" + group; - membership.clusterid = clusterId; - membership.clustertype = "content"; - membership.index = 2; - membership.retired = false; - return membership; - } - - private NodeRepositoryNode createNode(String nodename, String hostname, String clusterId, int group) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(nodename); - node.setParentHostname(hostname); - node.setState(NodeState.active); - node.setOwner(createOwner()); - node.setMembership(createMembership(clusterId, group)); - node.setType(NodeType.tenant); - - return node; + private Node createNode(String nodename, String hostname, String clusterId, int group) { + return Node.builder() + .hostname(nodename) + .parentHostname(hostname).state(Node.State.active) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .type(com.yahoo.config.provision.NodeType.tenant) + .clusterId(clusterId) + .group(String.valueOf(group)) + .clusterType(Node.ClusterType.content) + .build(); } - private NodeRepositoryNode createHost(String hostname, String switchName) { - NodeRepositoryNode node = new NodeRepositoryNode(); - node.setHostname(hostname); - node.setSwitchHostname(switchName); - node.setOwner(createOwner()); - node.setType(NodeType.host); - node.setMembership(createMembership("host", 0)); - return node; + private Node createHost(String hostname, String switchName) { + return Node.builder() + .hostname(hostname) + .switchHostname(switchName) + .owner(ApplicationId.from("mytenant", "myapp", "default")) + .type(com.yahoo.config.provision.NodeType.host) + .clusterId("host") + .group("0") + .build(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 7364723f5f0..c1358decf19 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -142,7 +142,7 @@ public class OsApiTest extends ControllerContainerTest { var targetVersion = nodeRepository().targetVersionsOf(zone).osVersion(application.nodeType()); for (Node node : nodeRepository().list(zone, application.id())) { var version = targetVersion.orElse(node.wantedOsVersion()); - nodeRepository().putNodes(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); + nodeRepository().putNodes(zone, Node.builder(node).currentOsVersion(version).wantedOsVersion(version).build()); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 4dd283cf5d7..6e70fb8c3cb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -70,7 +70,7 @@ public class VersionStatusTest { // Upgrade some config servers for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node upgradedNode = new Node.Builder(node).currentVersion(version1).build(); + Node upgradedNode = Node.builder(node).currentVersion(version1).build(); tester.configServer().nodeRepository().putNodes(zone.getId(), upgradedNode); break; } @@ -114,7 +114,7 @@ public class VersionStatusTest { Version ancientVersion = Version.fromString("5.1"); for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node downgradedNode = new Node.Builder(node).currentVersion(ancientVersion).build(); + Node downgradedNode = Node.builder(node).currentVersion(ancientVersion).build(); tester.configServer().nodeRepository().putNodes(zone.getId(), downgradedNode); break; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 63e39d8b398..4a470b7e2d2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -150,13 +150,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag METRICS_PROXY_MAX_HEAP_SIZE_IN_MB = defineIntFlag( - "metrics-proxy-max-heap-size-in-mb", 256, - List.of("hmusum"), "2021-03-01", "2021-07-15", - "JVM max heap size for metrics proxy in Mb", - "Takes effect when restarting metrics proxy", - CLUSTER_TYPE); - public static final UnboundStringFlag DEDICATED_CLUSTER_CONTROLLER_FLAVOR = defineStringFlag( "dedicated-cluster-controller-flavor", "", List.of("jonmv"), "2021-02-25", "2021-08-25", "Flavor as <vpu>-<memgb>-<diskgb> to use for dedicated cluster controller nodes", diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 695f0dd8659..5fe10f09f8a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -76,7 +76,7 @@ public class NodePrioritizer { // In dynamically provisioned zones, we can always take spare hosts since we can provision new on-demand, // NodeCandidate::compareTo will ensure that they will not be used until there is no room elsewhere. // In non-dynamically provisioned zones, we only allow allocating to spare hosts to replace failed nodes. - this.canAllocateToSpareHosts = dynamicProvisioning || isReplacement(nodesInCluster); + this.canAllocateToSpareHosts = dynamicProvisioning || isReplacement(nodesInCluster, clusterSpec.group()); // Do not allocate new nodes for exclusive deployments in dynamically provisioned zones: provision new host instead. this.canAllocateNew = requestedNodes instanceof NodeSpec.CountNodeSpec && (!dynamicProvisioning || !requestedNodes.isExclusive()); @@ -195,10 +195,13 @@ public class NodePrioritizer { } /** Returns whether we are allocating to replace a failed node */ - private boolean isReplacement(NodeList nodesInCluster) { - int failedNodesInCluster = nodesInCluster.failing().size() + nodesInCluster.state(Node.State.failed).size(); - if (failedNodesInCluster == 0) return false; - return ! requestedNodes.fulfilledBy(nodesInCluster.size() - failedNodesInCluster); + private boolean isReplacement(NodeList nodesInCluster, Optional<ClusterSpec.Group> group) { + NodeList nodesInGroup = group.map(ClusterSpec.Group::index) + .map(nodesInCluster::group) + .orElse(nodesInCluster); + int failedNodesInGroup = nodesInGroup.failing().size() + nodesInGroup.state(Node.State.failed).size(); + if (failedNodesInGroup == 0) return false; + return ! requestedNodes.fulfilledBy(nodesInGroup.size() - failedNodesInGroup); } /** diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index 2b180853d83..bdb93294cd1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -35,6 +35,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collection; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -49,13 +50,21 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static com.yahoo.config.provision.NodeResources.StorageType.remote; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; /** * Scenario tester with real node-repository data loaded from ZK snapshot file * + * How to use: + * + * 1. Copy /opt/vespa/conf/configserver-app/node-flavors.xml from config server to /tmp/node-flavors.xml + * 2. Copy /opt/vespa/var/zookeeper/version-2/snapshot.XXX from config server to /tmp/snapshot + * 3. Set capacities and specs according to the wanted scenario + * * @author valerijf */ public class RealDataScenarioTest { + private static final Logger log = Logger.getLogger(RealDataScenarioTest.class.getSimpleName()); @Ignore @@ -63,11 +72,11 @@ public class RealDataScenarioTest { public void test() { ProvisioningTester tester = new ProvisioningTester.Builder() .zone(new Zone(Cloud.builder().dynamicProvisioning(true).build(), SystemName.defaultSystem(), Environment.prod, RegionName.defaultName())) - .flavorsConfig(parseFlavors(Paths.get(System.getProperty("user.home"), ".flavors.xml"))) + .flavorsConfig(parseFlavors(Paths.get("/tmp/node-flavors.xml"))) .nameResolver(new DnsNameResolver()) .spareCount(1) .build(); - initFromZk(tester.nodeRepository(), Paths.get(System.getProperty("user.home"), "snapshot")); + initFromZk(tester.nodeRepository(), Paths.get("/tmp/snapshot")); ApplicationId app = ApplicationId.from("tenant", "app", "default"); Version version = Version.fromString("7.123.4"); @@ -91,10 +100,11 @@ public class RealDataScenarioTest { } private void deploy(ProvisioningTester tester, ApplicationId app, ClusterSpec[] specs, Capacity[] capacities) { + assertEquals("Equal capacity and spec count", capacities.length, specs.length); List<HostSpec> hostSpecs = IntStream.range(0, capacities.length) - .mapToObj(i -> tester.provisioner().prepare(app, specs[i], capacities[i], log::log).stream()) - .flatMap(s -> s) - .collect(Collectors.toList()); + .mapToObj(i -> tester.provisioner().prepare(app, specs[i], capacities[i], log::log)) + .flatMap(Collection::stream) + .collect(Collectors.toList()); NestedTransaction transaction = new NestedTransaction(); tester.provisioner().activate(hostSpecs, new ActivationContext(0), new ApplicationTransaction(new ProvisionLock(app, () -> {}), transaction)); transaction.commit(); @@ -144,4 +154,5 @@ public class RealDataScenarioTest { throw new UncheckedIOException(e); } } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 9ded28094d2..774350084ac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -148,11 +148,12 @@ public class NodeFailTester { } public static NodeFailTester withOneUndeployedApplication(Capacity capacity) { - NodeFailTester tester = new NodeFailTester(); + return withOneUndeployedApplication(capacity, ClusterSpec.request(ClusterSpec.Type.container, testCluster).vespaVersion("6.42").build()); + } - // Create applications - ClusterSpec clusterApp = ClusterSpec.request(ClusterSpec.Type.container, testCluster).vespaVersion("6.42").build(); - Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(app1, new MockDeployer.ApplicationContext(app1, clusterApp, capacity)); + public static NodeFailTester withOneUndeployedApplication(Capacity capacity, ClusterSpec spec) { + NodeFailTester tester = new NodeFailTester(); + Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(app1, new MockDeployer.ApplicationContext(app1, spec, capacity)); tester.initializeMaintainers(apps); return tester; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 7eff8af8b8d..dfeb82281e6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -377,6 +377,32 @@ public class NodeFailerTest { } @Test + public void node_failing_can_allocate_spare_to_replace_failed_node_in_group() { + NodeResources resources = new NodeResources(1, 20, 15, 1); + Capacity capacity = Capacity.from(new ClusterResources(4, 2, resources), false, true); + ClusterSpec spec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); + NodeFailTester tester = NodeFailTester.withOneUndeployedApplication(capacity, spec); + assertEquals("Test depends on this setting in NodeFailTester", 1, tester.nodeRepository.spareCount()); + tester.createAndActivateHosts(5, resources); // One extra - becomes designated spare + tester.activate(NodeFailTester.app1, spec, capacity); + + // Hardware failure is reported for host + NodeList activeNodes = tester.nodeRepository.nodes().list(Node.State.active); + Node downNode = activeNodes.owner(NodeFailTester.app1).first().get(); + Node downHost = activeNodes.parentOf(downNode).get(); + tester.tester.patchNode(downHost, (node) -> node.with(node.reports().withReport(badTotalMemorySizeReport))); + tester.suspend(downHost.hostname()); + tester.suspend(downNode.hostname()); + + // Node is failed and replaced + tester.runMaintainers(); + assertEquals(1, tester.deployer.redeployments); + NodeList failedOrActive = tester.nodeRepository.nodes().list(Node.State.active, Node.State.failed); + assertEquals(4, failedOrActive.state(Node.State.active).nodeType(NodeType.tenant).size()); + assertEquals(Set.of(downNode.hostname()), failedOrActive.state(Node.State.failed).nodeType(NodeType.tenant).hostnames()); + } + + @Test public void failing_ready_nodes() { NodeFailTester tester = NodeFailTester.withTwoApplications(); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 754cf99b00a..34bb88a19dd 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -803,12 +803,14 @@ public: mutable size_t prepare_set_tensor_cnt; mutable size_t complete_set_tensor_cnt; size_t clear_doc_cnt; + const Value* exp_tensor; MockDenseTensorAttribute(vespalib::stringref name, const AVConfig& cfg) : DenseTensorAttribute(name, cfg), prepare_set_tensor_cnt(0), complete_set_tensor_cnt(0), - clear_doc_cnt(0) + clear_doc_cnt(0), + exp_tensor() {} uint32_t clearDoc(DocId docid) override { ++clear_doc_cnt; @@ -816,6 +818,7 @@ public: } std::unique_ptr<PrepareResult> prepare_set_tensor(uint32_t docid, const Value& tensor) const override { ++prepare_set_tensor_cnt; + EXPECT_EQ(*exp_tensor, tensor); return std::make_unique<MockPrepareResult>(docid, tensor); } @@ -873,6 +876,7 @@ class TwoPhasePutTest : public AttributeWriterTest { public: Schema schema; DocBuilder builder; + vespalib::string doc_id; std::shared_ptr<MockDenseTensorAttribute> attr; std::unique_ptr<Value> tensor; @@ -880,6 +884,7 @@ public: : AttributeWriterTest(), schema(createTensorSchema(dense_tensor)), builder(schema), + doc_id("id:ns:searchdocument::1"), attr() { setup(2); @@ -889,6 +894,7 @@ public: attr->clear_doc_cnt = 0; tensor = make_tensor(TensorSpec(dense_tensor) .add({{"x", 0}}, 3).add({{"x", 1}}, 5)); + attr->exp_tensor = tensor.get(); } void expect_tensor_attr_calls(size_t exp_prepare_cnt, size_t exp_complete_cnt, @@ -901,13 +907,23 @@ public: return createTensorPutDoc(builder, *tensor); } Document::UP make_no_field_doc() { - return builder.startDocument("id:ns:searchdocument::1").endDocument(); + return builder.startDocument(doc_id).endDocument(); } Document::UP make_no_tensor_doc() { - return builder.startDocument("id:ns:searchdocument::1"). + return builder.startDocument(doc_id). startAttributeField("a1"). addTensor(std::unique_ptr<vespalib::eval::Value>()).endField().endDocument(); } + DocumentUpdate::UP make_assign_update() { + auto upd = std::make_unique<DocumentUpdate>(*builder.getDocumentTypeRepo(), + builder.getDocumentType(), + DocumentId(doc_id)); + TensorDataType tensor_type(vespalib::eval::ValueType::from_spec(dense_tensor)); + TensorFieldValue tensor_value(tensor_type); + tensor_value= SimpleValue::from_value(*tensor); + upd->addUpdate(FieldUpdate(upd->getType().getField("a1")).addUpdate(AssignValueUpdate(tensor_value))); + return upd; + } void expect_shared_executor_tasks(size_t exp_accepted_tasks) { auto stats = _shared.getStats(); EXPECT_EQ(exp_accepted_tasks, stats.acceptedTasks); @@ -958,6 +974,22 @@ TEST_F(TwoPhasePutTest, document_is_cleared_if_tensor_in_field_is_not_set) assertExecuteHistory({0}); } +TEST_F(TwoPhasePutTest, handles_assign_update_as_two_phase_put_when_specified_for_tensor_attribute) +{ + auto upd = make_assign_update(); + + DummyFieldUpdateCallback on_update; + update(1, *upd, 1, on_update); + expect_tensor_attr_calls(1, 1); + expect_shared_executor_tasks(1); + assertExecuteHistory({0}); + + update(2, *upd, 2, on_update); + expect_tensor_attr_calls(2, 2); + expect_shared_executor_tasks(2); + assertExecuteHistory({0, 0}); +} + ImportedAttributeVector::SP createImportedAttribute(const vespalib::string &name) diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index bcfdca9da19..a1c7b0a152d 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -1,10 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_usage_filter_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/size_literals.h> + #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> +#include <vespa/searchlib/attribute/address_space_components.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/size_literals.h> + +#include <vespa/log/log.h> +LOG_SETUP("attribute_usage_filter_test"); using proton::AttributeUsageFilter; using proton::AttributeUsageStats; @@ -24,15 +27,15 @@ class MyAttributeStats : public AttributeUsageStats { public: void triggerEnumStoreLimit() { - merge({ enumStoreOverLoad, - search::AddressSpaceUsage::defaultMultiValueUsage() }, + merge({ enumStoreOverLoad, + search::AddressSpaceComponents::default_multi_value_usage() }, "enumeratedName", "ready"); } void triggerMultiValueLimit() { - merge({ search::AddressSpaceUsage::defaultEnumStoreUsage(), - multiValueOverLoad }, + merge({ search::AddressSpaceComponents::default_enum_store_usage(), + multiValueOverLoad }, "multiValueName", "ready"); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp index f3da5486d3e..e7d6713441d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_usage_stats.h" +#include <vespa/searchlib/attribute/address_space_components.h> #include <iostream> namespace proton { AttributeUsageStats::AttributeUsageStats() - : _enumStoreUsage(search::AddressSpaceUsage::defaultEnumStoreUsage()), - _multiValueUsage(search::AddressSpaceUsage::defaultMultiValueUsage()) + : _enumStoreUsage(search::AddressSpaceComponents::default_enum_store_usage()), + _multiValueUsage(search::AddressSpaceComponents::default_multi_value_usage()) { } @@ -16,8 +17,8 @@ AttributeUsageStats::merge(const search::AddressSpaceUsage &usage, const vespalib::string &attributeName, const vespalib::string &subDbName) { - _enumStoreUsage.merge(usage.enumStoreUsage(), attributeName, subDbName); - _multiValueUsage.merge(usage.multiValueUsage(), attributeName, subDbName); + _enumStoreUsage.merge(usage.enum_store_usage(), attributeName, subDbName); + _multiValueUsage.merge(usage.multi_value_usage(), attributeName, subDbName); } std::ostream& diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index 2ffe3f2c901..ddf240ad90c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -63,8 +63,8 @@ convertAddressSpaceToSlime(const AddressSpace &addressSpace, Cursor &object) void convertAddressSpaceUsageToSlime(const AddressSpaceUsage &usage, Cursor &object) { - convertAddressSpaceToSlime(usage.enumStoreUsage(), object.setObject("enumStore")); - convertAddressSpaceToSlime(usage.multiValueUsage(), object.setObject("multiValue")); + convertAddressSpaceToSlime(usage.enum_store_usage(), object.setObject("enumStore")); + convertAddressSpaceToSlime(usage.multi_value_usage(), object.setObject("multiValue")); } void diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 218a37139bb..0abe73a2211 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -7,13 +7,14 @@ #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/searchcommon/attribute/attribute_utils.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/attribute_updater.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> -#include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/searchlib/tensor/prepare_result.h> -#include <vespa/searchcommon/attribute/attribute_utils.h> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/util/threadexecutor.h> #include <future> @@ -114,6 +115,21 @@ AttributeWriter::WriteContext::buildFieldPaths(const DocumentType &docType) } } +AttributeWriter::AttributeWithInfo::AttributeWithInfo() + : attribute(), + executor_id(), + use_two_phase_put_for_assign_updates(false) +{ +} + +AttributeWriter::AttributeWithInfo::AttributeWithInfo(search::AttributeVector* attribute_in, + ExecutorId executor_id_in) + : attribute(attribute_in), + executor_id(executor_id_in), + use_two_phase_put_for_assign_updates(use_two_phase_put_for_attribute(*attribute_in)) +{ +} + namespace { void @@ -340,10 +356,14 @@ private: std::promise<std::unique_ptr<PrepareResult>> _result_promise; public: - PreparePutTask(SerialNum serial_num_in, - uint32_t docid_in, + PreparePutTask(SerialNum serial_num, + uint32_t docid, const AttributeWriter::WriteField& field, std::shared_ptr<DocumentFieldExtractor> field_extractor); + PreparePutTask(SerialNum serial_num, + uint32_t docid, + AttributeVector& attr, + const FieldValue& field_value); ~PreparePutTask() override; void run() override; SerialNum serial_num() const { return _serial_num; } @@ -355,12 +375,12 @@ public: } }; -PreparePutTask::PreparePutTask(SerialNum serial_num_in, - uint32_t docid_in, +PreparePutTask::PreparePutTask(SerialNum serial_num, + uint32_t docid, const AttributeWriter::WriteField& field, std::shared_ptr<DocumentFieldExtractor> field_extractor) - : _serial_num(serial_num_in), - _docid(docid_in), + : _serial_num(serial_num), + _docid(docid), _attr(field.getAttribute()), _field_value(), _result_promise() @@ -370,6 +390,18 @@ PreparePutTask::PreparePutTask(SerialNum serial_num_in, _field_value.reset(value.release()); } +PreparePutTask::PreparePutTask(SerialNum serial_num, + uint32_t docid, + AttributeVector& attr, + const FieldValue& field_value) + : _serial_num(serial_num), + _docid(docid), + _attr(attr), + _field_value(field_value.clone()), + _result_promise() +{ +} + PreparePutTask::~PreparePutTask() = default; void @@ -604,13 +636,13 @@ AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr) _attrMap() { setupWriteContexts(); - setupAttriuteMapping(); + setupAttributeMapping(); } -void AttributeWriter::setupAttriuteMapping() { +void AttributeWriter::setupAttributeMapping() { for (auto attr : getWritableAttributes()) { vespalib::stringref name = attr->getName(); - _attrMap[name] = AttrWithId(attr, _attributeFieldWriter.getExecutorIdFromName(attr->getNamePrefix())); + _attrMap[name] = AttributeWithInfo(attr, _attributeFieldWriter.getExecutorIdFromName(attr->getNamePrefix())); } } @@ -664,6 +696,25 @@ AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum, OnWr } } +namespace { + +bool +is_single_assign_update(const FieldUpdate& update) +{ + return (update.getUpdates().size() == 1) && + (update.getUpdates()[0]->getType() == ValueUpdate::Assign) && + (static_cast<const AssignValueUpdate &>(*update.getUpdates()[0]).hasValue()); +} + +const FieldValue& +get_single_assign_update_field_value(const FieldUpdate& update) +{ + const auto& assign = static_cast<const AssignValueUpdate &>(*update.getUpdates()[0]); + return assign.getValue(); +} + +} + void AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) @@ -679,8 +730,8 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document for (const auto &fupd : upd.getUpdates()) { LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().data()); - auto found = _attrMap.find(fupd.getField().getName()); - AttributeVector * attrp = (found != _attrMap.end()) ? found->second.first : nullptr; + auto itr = _attrMap.find(fupd.getField().getName()); + AttributeVector * attrp = (itr != _attrMap.end()) ? itr->second.attribute : nullptr; onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (__builtin_expect(attrp == nullptr, false)) { LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().data()); @@ -688,10 +739,21 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document } // TODO: Check if we must use > due to multiple entries for same // document and attribute. - if (__builtin_expect(attrp->getStatus().getLastSyncToken() >= serialNum, false)) + if (__builtin_expect(attrp->getStatus().getLastSyncToken() >= serialNum, false)) { continue; - args[found->second.second.getId()]->_updates.emplace_back(attrp, &fupd); - LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attrp->getName().c_str()); + } + if (itr->second.use_two_phase_put_for_assign_updates && + is_single_assign_update(fupd)) { + auto prepare_task = std::make_unique<PreparePutTask>(serialNum, lid, *attrp, get_single_assign_update_field_value(fupd)); + auto complete_task = std::make_unique<CompletePutTask>(*prepare_task, onWriteDone); + LOG(debug, "About to handle assign update as two phase put for docid %u in attribute vector '%s'", + lid, attrp->getName().c_str()); + _shared_executor.execute(std::move(prepare_task)); + _attributeFieldWriter.executeTask(itr->second.executor_id, std::move(complete_task)); + } else { + args[itr->second.executor_id.getId()]->_updates.emplace_back(attrp, &fupd); + LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attrp->getName().c_str()); + } } // NOTE: The lifetime of the field update will be ensured by keeping the document update alive // in a operation done context object. @@ -708,8 +770,8 @@ void AttributeWriter::heartBeat(SerialNum serialNum) { for (auto entry : _attrMap) { - _attributeFieldWriter.execute(entry.second.second, - [serialNum, attr=entry.second.first]() + _attributeFieldWriter.execute(entry.second.executor_id, + [serialNum, attr=entry.second.attribute]() { applyHeartBeat(serialNum, *attr); }); } } @@ -737,8 +799,8 @@ void AttributeWriter::onReplayDone(uint32_t docIdLimit) { for (auto entry : _attrMap) { - _attributeFieldWriter.execute(entry.second.second, - [docIdLimit, attr = entry.second.first]() + _attributeFieldWriter.execute(entry.second.executor_id, + [docIdLimit, attr = entry.second.attribute]() { applyReplayDone(docIdLimit, *attr); }); } _attributeFieldWriter.sync(); @@ -749,8 +811,8 @@ void AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) { for (auto entry : _attrMap) { - _attributeFieldWriter.execute(entry.second.second, - [wantedLidLimit, serialNum, attr=entry.second.first]() + _attributeFieldWriter.execute(entry.second.executor_id, + [wantedLidLimit, serialNum, attr=entry.second.attribute]() { applyCompactLidSpace(wantedLidLimit, serialNum, *attr); }); } _attributeFieldWriter.sync(); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index 1c55f53aa5e..1cda579a645 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -67,16 +67,25 @@ public: bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } bool use_two_phase_put() const { return _use_two_phase_put; } }; + + struct AttributeWithInfo { + search::AttributeVector* attribute; + ExecutorId executor_id; + bool use_two_phase_put_for_assign_updates; + + AttributeWithInfo(); + AttributeWithInfo(search::AttributeVector* attribute_in, + ExecutorId executor_id_in); + }; private: - using AttrWithId = std::pair<search::AttributeVector *, ExecutorId>; - using AttrMap = vespalib::hash_map<vespalib::string, AttrWithId>; + using AttrMap = vespalib::hash_map<vespalib::string, AttributeWithInfo>; std::vector<WriteContext> _writeContexts; const DataType *_dataType; bool _hasStructFieldAttribute; AttrMap _attrMap; void setupWriteContexts(); - void setupAttriuteMapping(); + void setupAttributeMapping(); void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool allAttributes, OnWriteDoneType onWriteDone); diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 79e120d0683..0a4a562c5fb 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -6,22 +6,23 @@ #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/mapvalueupdate.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/attrvector.h> +#include <vespa/searchlib/attribute/multienumattribute.hpp> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multistringattribute.h> +#include <vespa/searchlib/attribute/multivalueattribute.hpp> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/attribute/singlenumericpostattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> -#include <vespa/searchlib/attribute/multivalueattribute.hpp> -#include <vespa/searchlib/attribute/multienumattribute.hpp> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/util/randomgenerator.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> +#include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> #include <cmath> @@ -2105,30 +2106,30 @@ AttributeTest::requireThatAddressSpaceUsageIsReported(const Config &config, bool AddressSpaceUsage after = attrPtr->getAddressSpaceUsage(); if (attrPtr->hasEnum()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has enum", attrName.c_str()); - EXPECT_EQUAL(before.enumStoreUsage().used(), 1u); - EXPECT_EQUAL(before.enumStoreUsage().dead(), 1u); - EXPECT_GREATER(after.enumStoreUsage().used(), before.enumStoreUsage().used()); - EXPECT_GREATER_EQUAL(after.enumStoreUsage().limit(), before.enumStoreUsage().limit()); - EXPECT_GREATER(after.enumStoreUsage().limit(), 4200000000u); + EXPECT_EQUAL(before.enum_store_usage().used(), 1u); + EXPECT_EQUAL(before.enum_store_usage().dead(), 1u); + EXPECT_GREATER(after.enum_store_usage().used(), before.enum_store_usage().used()); + EXPECT_GREATER_EQUAL(after.enum_store_usage().limit(), before.enum_store_usage().limit()); + EXPECT_GREATER(after.enum_store_usage().limit(), 4200000000u); } else { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): NOT enum", attrName.c_str()); - EXPECT_EQUAL(before.enumStoreUsage().used(), 0u); - EXPECT_EQUAL(before.enumStoreUsage().dead(), 0u); - EXPECT_EQUAL(after.enumStoreUsage(), before.enumStoreUsage()); - EXPECT_EQUAL(AddressSpaceUsage::defaultEnumStoreUsage(), after.enumStoreUsage()); + EXPECT_EQUAL(before.enum_store_usage().used(), 0u); + EXPECT_EQUAL(before.enum_store_usage().dead(), 0u); + EXPECT_EQUAL(after.enum_store_usage(), before.enum_store_usage()); + EXPECT_EQUAL(AddressSpaceComponents::default_enum_store_usage(), after.enum_store_usage()); } if (attrPtr->hasMultiValue()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has multi-value", attrName.c_str()); - EXPECT_EQUAL(before.multiValueUsage().used(), 1u); - EXPECT_EQUAL(before.multiValueUsage().dead(), 1u); - EXPECT_GREATER_EQUAL(after.multiValueUsage().used(), before.multiValueUsage().used()); - EXPECT_GREATER(after.multiValueUsage().limit(), before.multiValueUsage().limit()); - EXPECT_GREATER((1ull << 32), after.multiValueUsage().limit()); + EXPECT_EQUAL(before.multi_value_usage().used(), 1u); + EXPECT_EQUAL(before.multi_value_usage().dead(), 1u); + EXPECT_GREATER_EQUAL(after.multi_value_usage().used(), before.multi_value_usage().used()); + EXPECT_GREATER(after.multi_value_usage().limit(), before.multi_value_usage().limit()); + EXPECT_GREATER((1ull << 32), after.multi_value_usage().limit()); } else { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): NOT multi-value", attrName.c_str()); - EXPECT_EQUAL(before.multiValueUsage().used(), 0u); - EXPECT_EQUAL(after.multiValueUsage(), before.multiValueUsage()); - EXPECT_EQUAL(AddressSpaceUsage::defaultMultiValueUsage(), after.multiValueUsage()); + EXPECT_EQUAL(before.multi_value_usage().used(), 0u); + EXPECT_EQUAL(after.multi_value_usage(), before.multi_value_usage()); + EXPECT_EQUAL(AddressSpaceComponents::default_multi_value_usage(), after.multi_value_usage()); } } diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index a029373821a..be3013c051e 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -154,7 +154,7 @@ public: return status; } const Config &getConfig() const { return _v->getConfig(); } - AddressSpace getMultiValueAddressSpaceUsage() const {return _v->getAddressSpaceUsage().multiValueUsage(); } + AddressSpace getMultiValueAddressSpaceUsage() const {return _v->getAddressSpaceUsage().multi_value_usage(); } AddressSpace getMultiValueAddressSpaceUsage(const vespalib::string &prefix) { AddressSpace usage(getMultiValueAddressSpaceUsage()); LOG(info, "address space usage %s: used=%zu, dead=%zu, limit=%zu, usage=%12.8f", diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index bf56b476cc9..e7267e4f6a4 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_attribute OBJECT SOURCES + address_space_components.cpp address_space_usage.cpp attribute.cpp attribute_blueprint_factory.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp new file mode 100644 index 00000000000..74075367da9 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "address_space_components.h" +#include "i_enum_store.h" + +namespace search { + +using vespalib::AddressSpace; + +AddressSpace AddressSpaceComponents::default_enum_store_usage() { + return AddressSpace(0, 0, IEnumStore::InternalIndex::offsetSize()); +} + +AddressSpace AddressSpaceComponents::default_multi_value_usage() { + return AddressSpace(0, 0, (1ull << 32)); +} + +const vespalib::string AddressSpaceComponents::enum_store = "enum-store"; +const vespalib::string AddressSpaceComponents::multi_value = "multi-value"; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h new file mode 100644 index 00000000000..d52fe67abbd --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/address_space.h> + +namespace search { + +/** + * A set of components in an attribute vector that use address space. + */ +class AddressSpaceComponents { +public: + static vespalib::AddressSpace default_enum_store_usage(); + static vespalib::AddressSpace default_multi_value_usage(); + static const vespalib::string enum_store; + static const vespalib::string multi_value; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp index 692adaf3817..da2e376719c 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp @@ -1,33 +1,52 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "address_space_components.h" #include "address_space_usage.h" -#include "i_enum_store.h" namespace search { using vespalib::AddressSpace; AddressSpaceUsage::AddressSpaceUsage() - : _enumStoreUsage(defaultEnumStoreUsage()), - _multiValueUsage(defaultMultiValueUsage()) { + : _map() +{ +} + +AddressSpaceUsage::AddressSpaceUsage(const AddressSpace& enum_store_usage, + const AddressSpace& multi_value_usage) + : _map() +{ + // TODO: Remove this constructor and instead add usage for each relevant component explicit. + set(AddressSpaceComponents::enum_store, enum_store_usage); + set(AddressSpaceComponents::multi_value, multi_value_usage); +} + +void +AddressSpaceUsage::set(const vespalib::string& component, const vespalib::AddressSpace& usage) +{ + _map[component] = usage; } -AddressSpaceUsage::AddressSpaceUsage(const AddressSpace &enumStoreUsage_, - const AddressSpace &multiValueUsage_) - : _enumStoreUsage(enumStoreUsage_), - _multiValueUsage(multiValueUsage_) { +AddressSpace +AddressSpaceUsage::get(const vespalib::string& component) const +{ + auto itr = _map.find(component); + if (itr != _map.end()) { + return itr->second; + } + return AddressSpace(); } AddressSpace -AddressSpaceUsage::defaultEnumStoreUsage() +AddressSpaceUsage::enum_store_usage() const { - return AddressSpace(0, 0, IEnumStore::InternalIndex::offsetSize()); + return get(AddressSpaceComponents::enum_store); } AddressSpace -AddressSpaceUsage::defaultMultiValueUsage() +AddressSpaceUsage::multi_value_usage() const { - return AddressSpace(0, 0, (1ull << 32)); + return get(AddressSpaceComponents::multi_value); } -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_usage.h b/searchlib/src/vespa/searchlib/attribute/address_space_usage.h index 3e6ad1d3f8e..8698fa6d9ef 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_usage.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_usage.h @@ -2,28 +2,30 @@ #pragma once +#include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/address_space.h> +#include <unordered_map> namespace search { /** - * Represents the address space usage for enum store and multi value mapping. + * Represents the address space usage for a set of attribute vector components. */ class AddressSpaceUsage { private: - vespalib::AddressSpace _enumStoreUsage; - vespalib::AddressSpace _multiValueUsage; + using AddressSpaceMap = std::unordered_map<vespalib::string, vespalib::AddressSpace, vespalib::hash<vespalib::string>>; + AddressSpaceMap _map; public: AddressSpaceUsage(); - AddressSpaceUsage(const vespalib::AddressSpace &enumStoreUsage_, - const vespalib::AddressSpace &multiValueUsage_); - static vespalib::AddressSpace defaultEnumStoreUsage(); - static vespalib::AddressSpace defaultMultiValueUsage(); - const vespalib::AddressSpace &enumStoreUsage() const { return _enumStoreUsage; } - const vespalib::AddressSpace &multiValueUsage() const { return _multiValueUsage; } - + AddressSpaceUsage(const vespalib::AddressSpace& enum_store_usage, + const vespalib::AddressSpace& multi_value_usage); + void set(const vespalib::string& component, const vespalib::AddressSpace& usage); + vespalib::AddressSpace get(const vespalib::string& component) const; + vespalib::AddressSpace enum_store_usage() const; + vespalib::AddressSpace multi_value_usage() const; }; -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 08ddc01c15b..79baa10a381 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -1,10 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attributevector.h" +#include "address_space_components.h" #include "attribute_read_guard.h" #include "attributefilesavetarget.h" #include "attributeiterators.hpp" #include "attributesaver.h" +#include "attributevector.h" #include "attributevector.hpp" #include "floatbase.h" #include "interlock.h" @@ -14,14 +15,14 @@ #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/mapvalueupdate.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchcommon/attribute/attribute_utils.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/query/query_term_decoder.h> #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchlib/util/logutil.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/searchlib/util/logutil.h> -#include <vespa/searchcommon/attribute/attribute_utils.h> #include <thread> #include <vespa/log/log.h> @@ -222,22 +223,20 @@ AttributeVector::getEnumStoreValuesMemoryUsage() const return vespalib::MemoryUsage(); } -vespalib::AddressSpace -AttributeVector::getEnumStoreAddressSpaceUsage() const -{ - return AddressSpaceUsage::defaultEnumStoreUsage(); -} - -vespalib::AddressSpace -AttributeVector::getMultiValueAddressSpaceUsage() const +void +AttributeVector::populate_address_space_usage(AddressSpaceUsage& usage) const { - return AddressSpaceUsage::defaultMultiValueUsage(); + // TODO: Stop inserting defaults here when code using AddressSpaceUsage no longer require these two components. + usage.set(AddressSpaceComponents::enum_store, AddressSpaceComponents::default_enum_store_usage()); + usage.set(AddressSpaceComponents::multi_value, AddressSpaceComponents::default_multi_value_usage()); } AddressSpaceUsage AttributeVector::getAddressSpaceUsage() const { - return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), getMultiValueAddressSpaceUsage()); + AddressSpaceUsage usage; + populate_address_space_usage(usage); + return usage; } bool diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index a4294df8cb7..3c27f00a022 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -381,8 +381,7 @@ protected: } virtual vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const; - virtual vespalib::AddressSpace getEnumStoreAddressSpaceUsage() const; - virtual vespalib::AddressSpace getMultiValueAddressSpaceUsage() const; + virtual void populate_address_space_usage(AddressSpaceUsage& usage) const; public: DECLARE_IDENTIFIABLE_ABSTRACT(AttributeVector); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index 443433757b3..cee091b1ac8 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -64,7 +64,7 @@ protected: void insertNewUniqueValues(EnumStoreBatchUpdater& updater); virtual void considerAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) = 0; vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const override; - vespalib::AddressSpace getEnumStoreAddressSpaceUsage() const override; + void populate_address_space_usage(AddressSpaceUsage& usage) const override; public: EnumAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); ~EnumAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 164bb411061..643741b7eca 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -2,6 +2,7 @@ #pragma once +#include "address_space_components.h" #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchlib/attribute/enumattribute.h> #include <vespa/searchlib/attribute/enumstore.hpp> @@ -76,10 +77,11 @@ EnumAttribute<B>::getEnumStoreValuesMemoryUsage() const } template <typename B> -vespalib::AddressSpace -EnumAttribute<B>::getEnumStoreAddressSpaceUsage() const +void +EnumAttribute<B>::populate_address_space_usage(AddressSpaceUsage& usage) const { - return _enumStore.get_address_space_usage(); + B::populate_address_space_usage(usage); + usage.set(AddressSpaceComponents::enum_store, _enumStore.get_address_space_usage()); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index d36777a25a9..8e6e4914012 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -48,7 +48,7 @@ protected: **/ bool onAddDoc(DocId doc) override { (void) doc; return false; } - vespalib::AddressSpace getMultiValueAddressSpaceUsage() const override; + void populate_address_space_usage(AddressSpaceUsage& usage) const override; public: MultiValueAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 2e73909ea1e..190f73182f3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -2,6 +2,7 @@ #pragma once +#include "address_space_components.h" #include <vespa/searchlib/attribute/multivalueattribute.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -200,13 +201,13 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docVa } template <typename B, typename M> -vespalib::AddressSpace -MultiValueAttribute<B, M>::getMultiValueAddressSpaceUsage() const +void +MultiValueAttribute<B, M>::populate_address_space_usage(AddressSpaceUsage& usage) const { - return _mvMapping.getAddressSpaceUsage(); + B::populate_address_space_usage(usage); + usage.set(AddressSpaceComponents::multi_value, _mvMapping.getAddressSpaceUsage()); } - template <typename B, typename M> bool MultiValueAttribute<B, M>::addDoc(DocId & doc) diff --git a/slobrok/src/tests/service_map_history/service_map_history_test.cpp b/slobrok/src/tests/service_map_history/service_map_history_test.cpp index e88f326a99b..cdd1b2f7311 100644 --- a/slobrok/src/tests/service_map_history/service_map_history_test.cpp +++ b/slobrok/src/tests/service_map_history/service_map_history_test.cpp @@ -89,12 +89,12 @@ TEST(ServiceMapHistoryTest, full_inspection) { for (int i = 0; i < 1984; ++i) { auto name = fmt("key/%d/name", i); auto spec = fmt("tcp/host%d.domain.tld:19099", 10000+i); - p.update(ServiceMapping{name, spec}); + p.add(ServiceMapping{name, spec}); } EXPECT_EQ(p.currentGen(), GenCnt(1985)); - p.remove("key/666/name"); + p.remove(ServiceMapping{"key/666/name", "tcp/host10666.domain.tld:19099"}); EXPECT_EQ(p.currentGen(), GenCnt(1986)); - p.update(ServiceMapping{"key/1969/name", "tcp/woodstock:19069"}); + p.add(ServiceMapping{"key/1969/name", "tcp/woodstock:19069"}); EXPECT_EQ(p.currentGen(), GenCnt(1987)); auto map = dump(p); @@ -187,7 +187,7 @@ TEST(ServiceMapHistoryTest, handlers_test) { EXPECT_FALSE(cantCancel); handler1.got_update = false; - p.update(ServiceMapping{"foo", "bar"}); + p.add(ServiceMapping{"foo", "bar"}); EXPECT_FALSE(handler1.got_update); EXPECT_TRUE(handler2.got_update); EXPECT_FALSE(handler3.got_update); @@ -197,7 +197,7 @@ TEST(ServiceMapHistoryTest, handlers_test) { handler2.got_update = false; p.asyncGenerationDiff(&handler3, GenCnt(2)); EXPECT_FALSE(handler3.got_update); - p.remove("foo"); + p.remove(ServiceMapping{"foo", "bar"}); EXPECT_FALSE(handler1.got_update); EXPECT_FALSE(handler2.got_update); EXPECT_TRUE(handler3.got_update); diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp index 5f26608c294..4e1759f3a0d 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp +++ b/slobrok/src/vespa/slobrok/server/rpc_server_map.cpp @@ -27,8 +27,9 @@ RpcServerMap::lookup(const std::string & name) const std::unique_ptr<NamedService> RpcServerMap::remove(const std::string & name) { - _visible_map.remove(name); auto service = std::move(_myrpcsrv_map[name]); + auto spec = service->getSpec(); + _proxy.remove(ServiceMapping{name, spec}); _myrpcsrv_map.erase(name); return service; } @@ -67,7 +68,7 @@ RpcServerMap::add(NamedService *rpcsrv) LOG_ASSERT(_myrpcsrv_map.find(name) == _myrpcsrv_map.end()); removeReservation(name); - _visible_map.update(ServiceMapping{name, rpcsrv->getSpec()}); + _proxy.add(ServiceMapping{name, rpcsrv->getSpec()}); } void @@ -80,9 +81,8 @@ RpcServerMap::addNew(std::unique_ptr<ManagedRpcServer> rpcsrv) if (oldman) { const ReservedName *oldres = _reservations[name].get(); - _visible_map.remove(name); - const std::string &spec = rpcsrv->getSpec(); + _proxy.remove(ServiceMapping{name, spec}); const std::string &oldname = oldman->getName(); const std::string &oldspec = oldman->getSpec(); if (spec != oldspec) { @@ -133,11 +133,7 @@ RpcServerMap::getReservation(const std::string &name) const { return (found == _reservations.end()) ? nullptr : found->second.get(); } -RpcServerMap::RpcServerMap() - : _myrpcsrv_map(), - _reservations() -{ -} +RpcServerMap::RpcServerMap() = default; RpcServerMap::~RpcServerMap() = default; diff --git a/slobrok/src/vespa/slobrok/server/rpc_server_map.h b/slobrok/src/vespa/slobrok/server/rpc_server_map.h index 7b6c068b791..ddc85395ecd 100644 --- a/slobrok/src/vespa/slobrok/server/rpc_server_map.h +++ b/slobrok/src/vespa/slobrok/server/rpc_server_map.h @@ -3,6 +3,7 @@ #include "named_service.h" #include "service_map_history.h" +#include "proxy_map_source.h" #include <memory> #include <string> @@ -28,9 +29,9 @@ class RpcServerMap private: using ManagedRpcServerMap = std::unordered_map<std::string, std::unique_ptr<ManagedRpcServer>>; using ReservedNameMap = std::unordered_map<std::string, std::unique_ptr<ReservedName>>; - ServiceMapHistory _visible_map; ManagedRpcServerMap _myrpcsrv_map; ReservedNameMap _reservations; + ProxyMapSource _proxy; static bool match(const char *name, const char *pattern); @@ -39,8 +40,7 @@ private: public: typedef std::vector<const NamedService *> RpcSrvlist; - ServiceMapHistory& localView() { return _visible_map; } - ServiceMapHistory& visibleMap() { return _visible_map; } + MapSource &proxy() { return _proxy; } ManagedRpcServer *lookupManaged(const std::string & name) const; diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.cpp b/slobrok/src/vespa/slobrok/server/rpchooks.cpp index 51cd3d65857..2fefce1d474 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.cpp +++ b/slobrok/src/vespa/slobrok/server/rpchooks.cpp @@ -43,6 +43,8 @@ public: RPCHooks::RPCHooks(SBEnv &env, RpcServerMap& rpcsrvmap, RpcServerManager& rpcsrvman) : _env(env), _rpcsrvmap(rpcsrvmap), _rpcsrvmanager(rpcsrvman), + _globalHistory(env.globalHistory()), + _localHistory(env.localHistory()), _cnts(Metrics::zero()), _m_reporter() { @@ -493,7 +495,7 @@ RPCHooks::rpc_incrementalFetch(FRT_RPCRequest *req) uint32_t msTimeout = args[1]._intval32; req->getStash().create<IncrementalFetch>(_env.getSupervisor(), req, - _rpcsrvmap.visibleMap(), gencnt).invoke(msTimeout); + _globalHistory, gencnt).invoke(msTimeout); } void RPCHooks::rpc_fetchLocalView(FRT_RPCRequest *req) { @@ -502,7 +504,7 @@ void RPCHooks::rpc_fetchLocalView(FRT_RPCRequest *req) { vespalib::GenCnt gencnt(args[0]._intval32); uint32_t msTimeout = args[1]._intval32; req->getStash().create<IncrementalFetch>(_env.getSupervisor(), req, - _rpcsrvmap.localView(), gencnt).invoke(msTimeout); + _localHistory, gencnt).invoke(msTimeout); } diff --git a/slobrok/src/vespa/slobrok/server/rpchooks.h b/slobrok/src/vespa/slobrok/server/rpchooks.h index 66f0ed892f3..a41e473b183 100644 --- a/slobrok/src/vespa/slobrok/server/rpchooks.h +++ b/slobrok/src/vespa/slobrok/server/rpchooks.h @@ -12,6 +12,7 @@ namespace slobrok { class SBEnv; class RpcServerMap; class RpcServerManager; +class ServiceMapHistory; /** * @class RPCHooks @@ -41,6 +42,8 @@ private: SBEnv &_env; RpcServerMap &_rpcsrvmap; RpcServerManager &_rpcsrvmanager; + ServiceMapHistory &_globalHistory; + ServiceMapHistory &_localHistory; Metrics _cnts; std::unique_ptr<FNET_Task> _m_reporter; diff --git a/slobrok/src/vespa/slobrok/server/sbenv.cpp b/slobrok/src/vespa/slobrok/server/sbenv.cpp index 2bf8e57cb26..78c34f1bdb4 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.cpp +++ b/slobrok/src/vespa/slobrok/server/sbenv.cpp @@ -115,6 +115,7 @@ SBEnv::SBEnv(const ConfigShim &shim) _rpcsrvmap() { srandom(time(nullptr) ^ getpid()); + _rpcsrvmap.proxy().registerListener(_globalVisibleHistory); _rpcHooks.initRPC(getSupervisor()); } diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index 1050bd1359a..6836715854a 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -54,6 +54,7 @@ private: vespalib::SimpleHealthProducer _health; MetricsProducer _metrics; vespalib::SimpleComponentConfigProducer _components; + ServiceMapHistory _globalVisibleHistory; public: explicit SBEnv(const ConfigShim &shim); @@ -71,6 +72,14 @@ public: ExchangeManager _exchanger; RpcServerMap _rpcsrvmap; + ServiceMapHistory& globalHistory() { + return _globalVisibleHistory; + } + + ServiceMapHistory& localHistory() { + return _globalVisibleHistory; + } + const std::string & mySpec() const { return _me; } bool isSuspended() const { return false; } diff --git a/slobrok/src/vespa/slobrok/server/service_map_history.cpp b/slobrok/src/vespa/slobrok/server/service_map_history.cpp index 3882a41bde5..5a2f8c6ff43 100644 --- a/slobrok/src/vespa/slobrok/server/service_map_history.cpp +++ b/slobrok/src/vespa/slobrok/server/service_map_history.cpp @@ -82,24 +82,29 @@ bool ServiceMapHistory::cancel(DiffCompletionHandler *handler) { return (removed > 0); } -void ServiceMapHistory::remove(const vespalib::string &name) { +void ServiceMapHistory::remove(const ServiceMapping &mapping) { { std::lock_guard guard(_lock); - auto iter = _map.find(name); + auto iter = _map.find(mapping.name); if (iter == _map.end()) { - LOG(warning, "already removed: %s", name.c_str()); - // already removed - return; + LOG(debug, "already removed: %s", mapping.name.c_str()); + return; // already removed } + LOG_ASSERT(iter->second == mapping.spec); _map.erase(iter); - _log.add(name); + _log.add(mapping.name); } notify_updated(); } -void ServiceMapHistory::update(const ServiceMapping &mapping) { +void ServiceMapHistory::add(const ServiceMapping &mapping) { { std::lock_guard guard(_lock); + auto iter = _map.find(mapping.name); + if (iter != _map.end() && iter->second == mapping.spec) { + // already ok + return; + } _map.insert_or_assign(mapping.name, mapping.spec); _log.add(mapping.name); } diff --git a/slobrok/src/vespa/slobrok/server/service_map_history.h b/slobrok/src/vespa/slobrok/server/service_map_history.h index e1f20e0d8a4..9026586b945 100644 --- a/slobrok/src/vespa/slobrok/server/service_map_history.h +++ b/slobrok/src/vespa/slobrok/server/service_map_history.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/arrayqueue.hpp> #include <map> #include <mutex> +#include "map_listener.h" #include "service_mapping.h" #include "map_diff.h" @@ -16,8 +17,7 @@ namespace slobrok { * @class ServiceMapHistory * @brief API to generate incremental updates for a collection of name->spec mappings **/ - -class ServiceMapHistory +class ServiceMapHistory : public MapListener { public: using Generation = vespalib::GenCnt; @@ -66,6 +66,11 @@ public: ~ServiceMapHistory(); /** + * Get diff from generation fromGen (sync version). + **/ + MapDiff makeDiffFrom(const Generation &fromGen) const; + + /** * Ask for notification when the history has changes newer than fromGen. * Note that if there are any changes in the history already, the callback * will happen immediately (inside asyncGenerationDiff). @@ -78,17 +83,14 @@ public: **/ bool cancel(DiffCompletionHandler *handler); - /** add or update name->spec mapping */ - void update(const ServiceMapping &mapping); + /** add name->spec mapping */ + void add(const ServiceMapping &mapping) override; /** remove mapping for name */ - void remove(const vespalib::string &name); + void remove(const ServiceMapping &mapping) override; /** For unit testing only: */ Generation currentGen() const { return myGen(); } - -private: - MapDiff makeDiffFrom(const Generation &fromGen) const; }; //----------------------------------------------------------------------------- diff --git a/vespajlib/src/main/java/com/yahoo/osgi/maven/ProjectBundleClassPaths.java b/vespajlib/src/main/java/com/yahoo/osgi/maven/ProjectBundleClassPaths.java deleted file mode 100644 index 5d64548e4b9..00000000000 --- a/vespajlib/src/main/java/com/yahoo/osgi/maven/ProjectBundleClassPaths.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.osgi.maven; - -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -/** - * Represents the bundles in a maven project and the classpath elements - * corresponding to code that would end up in the bundle. - * - * @author Tony Vaagenes - * @author bjorncs - */ - -public class ProjectBundleClassPaths { - public static final String CLASSPATH_MAPPINGS_FILENAME = "bundle-plugin.bundle-classpath-mappings.json"; - - public final BundleClasspathMapping mainBundle; - public final List<BundleClasspathMapping> providedDependencies; - - public ProjectBundleClassPaths(BundleClasspathMapping mainBundle, - List<BundleClasspathMapping> providedDependencies) { - this.mainBundle = mainBundle; - this.providedDependencies = providedDependencies; - } - - public static void save(Path path, ProjectBundleClassPaths mappings) throws IOException { - Files.createDirectories(path.getParent()); - try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(path))) { - save(out, mappings); - } - } - - static void save(OutputStream out, ProjectBundleClassPaths mappings) throws IOException { - Slime slime = new Slime(); - Cursor rootCursor = slime.setObject(); - Cursor mainBundleCursor = rootCursor.setObject("mainBundle"); - BundleClasspathMapping.save(mainBundleCursor, mappings.mainBundle); - Cursor dependenciesCursor = rootCursor.setArray("providedDependencies"); - mappings.providedDependencies - .forEach(d -> BundleClasspathMapping.save(dependenciesCursor.addObject(), d)); - new JsonFormat(false).encode(out, slime); - } - - public static ProjectBundleClassPaths load(Path path) throws IOException { - byte[] bytes = Files.readAllBytes(path); - return load(bytes); - } - - static ProjectBundleClassPaths load(byte[] bytes) { - Inspector inspector = SlimeUtils.jsonToSlime(bytes).get(); - BundleClasspathMapping mainBundle = BundleClasspathMapping.load(inspector.field("mainBundle")); - Inspector dependenciesInspector = inspector.field("providedDependencies"); - List<BundleClasspathMapping> providedDependencies = new ArrayList<>(); - for (int i = 0; i < dependenciesInspector.entries(); i++) { - providedDependencies.add(BundleClasspathMapping.load(dependenciesInspector.entry(i))); - } - return new ProjectBundleClassPaths(mainBundle, providedDependencies); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ProjectBundleClassPaths that = (ProjectBundleClassPaths) o; - return Objects.equals(mainBundle, that.mainBundle) && - Objects.equals(providedDependencies, that.providedDependencies); - } - - @Override - public int hashCode() { - return Objects.hash(mainBundle, providedDependencies); - } - - public static class BundleClasspathMapping { - public final String bundleSymbolicName; - public final List<String> classPathElements; - - public BundleClasspathMapping(String bundleSymbolicName, - List<String> classPathElements) { - this.bundleSymbolicName = bundleSymbolicName; - this.classPathElements = classPathElements; - } - - static void save(Cursor rootCursor, BundleClasspathMapping mapping) { - rootCursor.setString("bundleSymbolicName", mapping.bundleSymbolicName); - Cursor arrayCursor = rootCursor.setArray("classPathElements"); - mapping.classPathElements.forEach(arrayCursor::addString); - } - - static BundleClasspathMapping load(Inspector inspector) { - String bundleSymoblicName = inspector.field("bundleSymbolicName").asString(); - Inspector elementsInspector = inspector.field("classPathElements"); - List<String> classPathElements = new ArrayList<>(); - for (int i = 0; i < elementsInspector.entries(); i++) { - classPathElements.add(elementsInspector.entry(i).asString()); - } - return new BundleClasspathMapping(bundleSymoblicName, classPathElements); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BundleClasspathMapping that = (BundleClasspathMapping) o; - return Objects.equals(bundleSymbolicName, that.bundleSymbolicName) && - Objects.equals(classPathElements, that.classPathElements); - } - - @Override - public int hashCode() { - return Objects.hash(bundleSymbolicName, classPathElements); - } - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/osgi/maven/ProjectBundleClassPathsTest.java b/vespajlib/src/test/java/com/yahoo/osgi/maven/ProjectBundleClassPathsTest.java deleted file mode 100644 index 44a6a2eee8d..00000000000 --- a/vespajlib/src/test/java/com/yahoo/osgi/maven/ProjectBundleClassPathsTest.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.osgi.maven; - -import com.yahoo.osgi.maven.ProjectBundleClassPaths.BundleClasspathMapping; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; - -/** - * @author bjorncs - */ -public class ProjectBundleClassPathsTest { - - @Test - public void bundle_classpaths_serializes_correctly_to_json() throws IOException { - ProjectBundleClassPaths projectBundleClassPaths = - new ProjectBundleClassPaths( - new BundleClasspathMapping("main-bundle-name", asList("classpath-elem-0-1", "classpath-elem-0-2")), - asList( - new BundleClasspathMapping( - "main-bundle-dep1", - asList("classpath-elem-1-1", "classpath-elem-1-2")), - new BundleClasspathMapping( - "main-bundle-dep2", - asList("classpath-elem-2-1", "classpath-elem-2-2")))); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ProjectBundleClassPaths.save(out, projectBundleClassPaths); - ProjectBundleClassPaths deserialized = ProjectBundleClassPaths.load(out.toByteArray()); - assertEquals(projectBundleClassPaths, deserialized); - } - -} diff --git a/vespalib/src/vespa/vespalib/util/address_space.cpp b/vespalib/src/vespa/vespalib/util/address_space.cpp index 113e4ba1478..1f6874e3e2a 100644 --- a/vespalib/src/vespa/vespalib/util/address_space.cpp +++ b/vespalib/src/vespa/vespalib/util/address_space.cpp @@ -6,6 +6,13 @@ namespace vespalib { +AddressSpace::AddressSpace() + : _used(0), + _dead(0), + _limit(0) +{ +} + AddressSpace::AddressSpace(size_t used_, size_t dead_, size_t limit_) : _used(used_), _dead(dead_), diff --git a/vespalib/src/vespa/vespalib/util/address_space.h b/vespalib/src/vespa/vespalib/util/address_space.h index 98ffdac9599..8eb3ba811f9 100644 --- a/vespalib/src/vespa/vespalib/util/address_space.h +++ b/vespalib/src/vespa/vespalib/util/address_space.h @@ -18,6 +18,7 @@ private: size_t _limit; public: + AddressSpace(); AddressSpace(size_t used_, size_t dead_, size_t limit_); size_t used() const { return _used; } size_t dead() const { return _dead; } |