diff options
57 files changed, 660 insertions, 741 deletions
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 3020184fd35..864c8d6fe9c 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -59,6 +59,7 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { try { Artifacts.ArtifactSet artifactSet = Artifacts.getArtifacts(project); warnOnUnsupportedArtifacts(artifactSet.getNonJarArtifacts()); + warnIfInternalContainerArtifactsAreIncluded(artifactSet.getJarArtifactsToInclude()); List<Export> exportedPackagesFromProvidedJars = exportedPackagesAggregated( artifactSet.getJarArtifactsProvided().stream().map(Artifact::getFile).collect(Collectors.toList())); @@ -180,6 +181,23 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { artifact.getId(), artifact.getType()))); } + // TODO: fail the build by throwing a MojoExecutionException + private void warnIfInternalContainerArtifactsAreIncluded(Collection<Artifact> includedArtifacts) throws MojoExecutionException { + /* In most cases it's sufficient to test for 'component', as it's the lowest level container artifact, + * Embedding container artifacts will cause class loading issues at runtime, because the classes will + * not be equal to those seen by the framework (e.g. AbstractComponent). + */ + if (includedArtifacts.stream().anyMatch(this::isJdiscComponentArtifact)) { + getLog().warn("This project includes the 'com.yahoo.vespa:component' artifact in compile scope." + + " It must be set to scope 'provided' to avoid resource leaks in your application at runtime." + + " The build will fail on a future Vespa version unless this is fixed."); + } + } + + private boolean isJdiscComponentArtifact(Artifact a) { + return a.getArtifactId().equals("component") && a.getGroupId().equals("com.yahoo.vespa"); + } + private PackageTally getProjectClassesTally() { File outputDirectory = new File(project.getBuild().getOutputDirectory()); diff --git a/cloud-tenant-base/is-base-pom-module.txt b/cloud-tenant-base/is-base-pom-module.txt new file mode 100644 index 00000000000..ea1e494bc0b --- /dev/null +++ b/cloud-tenant-base/is-base-pom-module.txt @@ -0,0 +1 @@ +Used to skip 'hosted-build-vespa-application' profile when building hosted-tenant-base itself
\ No newline at end of file diff --git a/cloud-tenant-base/pom.xml b/cloud-tenant-base/pom.xml index 8ffa6be4f9c..99e2c5badc6 100644 --- a/cloud-tenant-base/pom.xml +++ b/cloud-tenant-base/pom.xml @@ -39,345 +39,17 @@ <properties> <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + <bundlePluginExtraTestProvidedDependencies>com.yahoo.vespa:cloud-tenant-cd</bundlePluginExtraTestProvidedDependencies> </properties> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-dependency-versions</artifactId> - <version>${vespaversion}</version> - <type>pom</type> - <scope>import</scope> - </dependency> - - <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - <version>${junit.version}</version> - </dependency> - - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <version>${junit.version}</version> - </dependency> - </dependencies> - </dependencyManagement> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>container</artifactId> - <version>${vespaversion}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> + <artifactId>cloud-tenant-cd</artifactId> <version>${vespaversion}</version> <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>cloud-tenant-cd</artifactId> - <version>${test-framework.version}</version> - <scope>runtime</scope> </dependency> </dependencies> - <profiles> - <profile> - <!-- Build *-fat-test.jar file that includes all non-test classes and resources - that are part of the class path during test and and test.jar that includes - all test classes and resources, and put it inside a zip: - 1. application classes and resources - 2. test classes and resources - 3. classes and resources in all dependencies of both (1) and (2) - 4. copy the fat-test-jar and test-jar to application-test/artifacts directory - 5. zip application-test --> - <id>fat-test-application</id> - <build> - <plugins> - <plugin> - <!-- dependencies, see (3) above --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <version>3.1.1</version> - <executions> - <execution> - <!-- JAR-like dependencies --> - <id>unpack-dependencies</id> - <phase>prepare-package</phase> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <configuration> - <includeTypes>jar,test-jar</includeTypes> - <outputDirectory>target/fat-test-classes</outputDirectory> - <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and - bcprov-jdk15on-1.58.jar are pulled in via - container-dev and both contains the same set of - bouncycastle signature files in META-INF: - BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and - BC2048KE.SF. By merging any of these two with any - other JAR file like we're doing here, the signatures - are wrong. Worse, what we're doing is WRONG but not - yet fatal. - - The symptom of this happening is that the tester fails - to load the SystemTest class(!?), and subsequently - tries to run all test-like files in the fat test JAR. - - The solution is to exclude such files. This happens - automatically with maven-assembly-plugin. --> - <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> - </configuration> - </execution> - <execution> - <!-- non-JAR-like dependencies --> - <id>non-jar-dependencies</id> - <phase>prepare-package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - <configuration> - <excludeTypes>jar,test-jar</excludeTypes> - <outputDirectory>target/fat-test-classes</outputDirectory> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <artifactId>maven-resources-plugin</artifactId> - <version>3.1.0</version> - <executions> - <execution> - <id>copy-resources</id> - <phase>prepare-package</phase> - <goals> - <goal>copy-resources</goal> - </goals> - <configuration> - <outputDirectory>target/fat-test-classes</outputDirectory> - <resources> - <!-- application classes and resources, see 1. above --> - <resource> - <directory>target/classes</directory> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>3.1.0</version> - <executions> - <execution> - <id>fat-test-jar</id> - <phase>package</phase> - <goals> - <goal>jar</goal> - </goals> - <configuration> - <classesDirectory>target/fat-test-classes</classesDirectory> - <classifier>fat-test</classifier> - </configuration> - </execution> - <execution> - <id>test-jar</id> - <phase>package</phase> - <goals> - <goal>test-jar</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>attach-artifact</id> - <phase>package</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <tasks> - <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> - <copy file="target/${project.artifactId}-fat-test.jar" - todir="target/application-test/artifacts/" /> - - <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> - <copy file="target/${project.artifactId}-tests.jar" - todir="target/application-test/artifacts/" /> - - <!-- zip application-test, see 5. above --> - <zip destfile="target/application-test.zip" - basedir="target/application-test/" /> - </tasks> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - - <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version --> - <id>set-vespa-compile-version</id> - <activation> - <property> - <name>vespa.compile.version</name> - </property> - </activation> - <properties> - <vespaversion>${vespa.compile.version}</vespaversion> - </properties> - </profile> - - <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version --> - <id>set-vespa-runtime-version</id> - <activation> - <property> - <name>vespa.runtime.version</name> - </property> - </activation> - <properties> - <vespaVersion>${vespa.runtime.version}</vespaVersion> - </properties> - </profile> - </profiles> - - <build> - <finalName>${project.artifactId}</finalName> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <groups>${test.categories}</groups> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - <systemPropertyVariables> - <application>${application}</application> - <tenant>${tenant}</tenant> - <instance>${instance}</instance> - <environment>${environment}</environment> - <region>${region}</region> - <endpoint>${endpoint}</endpoint> - <apiKeyFile>${apiKeyFile}</apiKeyFile> - <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> - <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> - <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> - </systemPropertyVariables> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - </configuration> - </plugin> - </plugins> - </pluginManagement> - - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <version>3.0.0-M2</version> - <executions> - <execution> - <id>enforce-java</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireJavaVersion> - <version>[11, )</version> - </requireJavaVersion> - <requireMavenVersion> - <version>[3.5, )</version> - </requireMavenVersion> - </rules> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>${maven-compiler-plugin.version}</version> - <configuration> - <source>${target_jdk_version}</source> - <target>${target_jdk_version}</target> - <showWarnings>true</showWarnings> - <showDeprecation>true</showDeprecation> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-maven-plugin</artifactId> - <version>${vespaversion}</version> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-application-maven-plugin</artifactId> - <version>${vespaversion}</version> - <executions> - <execution> - <goals> - <goal>packageApplication</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${vespaversion}</version> - <extensions>true</extensions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - </plugin> - </plugins> - </build> </project> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 6c0c2c1260f..db160c5f6ae 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -95,6 +95,9 @@ public interface ModelContext { /// Default setting for the gc-options attribute if not specified explicit by application String jvmGCOptions(); + // Select sequencer type use while feeding. + String feedSequencerType(); + boolean useDistributorBtreeDb(); boolean useThreePhaseUpdates(); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 2232e57a06f..181c4ec100a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,6 +43,7 @@ public class TestProperties implements ModelContext.Properties { private double threadPoolSizeFactor = 0.0; private double queueSizeFactor = 0.0; private String jvmGCOptions = null; + private String sequencerType = "LATENCY"; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; private ApplicationRoles applicationRoles; @@ -57,7 +58,7 @@ public class TestProperties implements ModelContext.Properties { @Override public Zone zone() { return zone; } @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } @Override public String jvmGCOptions() { return jvmGCOptions; } - + @Override public String feedSequencerType() { return sequencerType; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return false; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @@ -78,6 +79,10 @@ public class TestProperties implements ModelContext.Properties { jvmGCOptions = gcOptions; return this; } + public TestProperties setFeedSequencerType(String type) { + sequencerType = type; + return this; + } public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 6cb45c4559b..3efc26913a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -43,7 +44,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private final boolean flushOnShutdown; /** If this is set up for streaming search, it is modelled as one search cluster per search definition */ - private Map<String, AbstractSearchCluster> clusters = new TreeMap<>(); + private final Map<String, AbstractSearchCluster> clusters = new TreeMap<>(); /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ private IndexedSearchCluster indexedCluster; @@ -55,10 +56,11 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private Double visibilityDelay = 0.0; /** The search nodes of this if it does not have an indexed cluster */ - private List<SearchNode> nonIndexed = new ArrayList<>(); + private final List<SearchNode> nonIndexed = new ArrayList<>(); - private Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); + private final Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); + private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -89,6 +91,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, + deployState.getProperties(), documentDefinitions, globallyDistributedDocuments, getFlushOnShutdown(flushOnShutdownElem, deployState), @@ -174,8 +177,17 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot } } + private static ProtonConfig.Indexing.Optimize.Enum convertFeedSequencerType(String sequencerType) { + try { + return ProtonConfig.Indexing.Optimize.Enum.valueOf(sequencerType); + } catch (Throwable t) { + return ProtonConfig.Indexing.Optimize.Enum.LATENCY; + } + } + private ContentSearchCluster(AbstractConfigProducer parent, String clusterName, + ModelContext.Properties featureFlags, Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, boolean flushOnShutdown, @@ -187,6 +199,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot this.globallyDistributedDocuments = globallyDistributedDocuments; this.flushOnShutdown = flushOnShutdown; this.combined = combined; + feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); } public void setVisibilityDelay(double delay) { @@ -379,6 +392,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot if (hasAnyNonIndexedCluster) { builder.feeding.concurrency(builder.feeding.build().concurrency() * 2); } + builder.indexing.optimize(feedSequencerType); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index b4fe4175707..5d9b5e3dbce 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -760,6 +760,38 @@ public class ContentBuilderTest extends DomBuilderTest { } } + private void verifyFeedSequencer(String input, String expceted) { + String hostedXml = "<services>" + + "<content version='1.0' id='search'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes count='1'/>" + + "</content>" + + "</services>"; + + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setFeedSequencerType(input)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(hostedXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(expceted, config.indexing().optimize().toString()); + + } + + @Test + public void ensureFeedSequencerIsControlledByFlag() { + verifyFeedSequencer("LATENCY", "LATENCY"); + verifyFeedSequencer("ADAPTIVE", "ADAPTIVE"); + verifyFeedSequencer("THROUGHPUT", "THROUGHPUT"); + verifyFeedSequencer("THOUGHPUT", "LATENCY"); + verifyFeedSequencer("adaptive", "LATENCY"); + + } + @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index 58ae8b95e97..fc267bbac29 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -45,10 +45,16 @@ public enum NodeType { this.description = description; } + // TODO: Remove when no version older than 7.247 is live + @Deprecated public boolean isDockerHost() { return !childNodeTypes.isEmpty(); } + public boolean isHost() { + return !childNodeTypes.isEmpty(); + } + public String description() { return description; } @@ -66,7 +72,7 @@ public enum NodeType { * @throws IllegalStateException if this type is not a host */ public List<NodeType> childNodeTypes() { - if (! isDockerHost()) + if (! isHost()) throw new IllegalStateException(this + " has no children"); return childNodeTypes; } diff --git a/config/src/tests/configgen/CMakeLists.txt b/config/src/tests/configgen/CMakeLists.txt index 533eeb46f70..663e265530f 100644 --- a/config/src/tests/configgen/CMakeLists.txt +++ b/config/src/tests/configgen/CMakeLists.txt @@ -4,11 +4,9 @@ vespa_add_executable(config_configgen_test_app TEST configgen.cpp DEPENDS config_cloudconfig - $<TARGET_OBJECTS:config_tests_configgen_config> - AFTER - config_tests_configgen_config ) vespa_add_test(NAME config_configgen_test_app COMMAND config_configgen_test_app) +vespa_generate_config(config_configgen_test_app ../../test/resources/configdefinitions/motd.def) vespa_add_executable(config_vector_inserter_test_app TEST SOURCES vector_inserter.cpp @@ -28,13 +26,5 @@ vespa_add_executable(config_value_converter_test_app TEST value_converter.cpp DEPENDS config_cloudconfig - $<TARGET_OBJECTS:config_tests_configgen_config> - AFTER - config_tests_configgen_config ) vespa_add_test(NAME config_value_converter_test_app COMMAND config_value_converter_test_app) - -vespa_add_library(config_tests_configgen_config OBJECT -) - -vespa_generate_config(config_tests_configgen_config ../../test/resources/configdefinitions/motd.def) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 74a9e72e255..b252ee3ef56 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -154,6 +154,7 @@ public class ModelContextImpl implements ModelContext { private final double threadPoolSizeFactor; private final double queueSizefactor; private final String jvmGCOPtions; + private final String feedSequencer; private final Optional<AthenzDomain> athenzDomain; private final Optional<ApplicationRoles> applicationRoles; private final int jdiscHealthCheckProxyClientTimeout; @@ -197,6 +198,8 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + feedSequencer = Flags.FEED_SEQUENCER_TYPE.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; this.applicationRoles = applicationRoles; jdiscHealthCheckProxyClientTimeout = Flags.JDISC_HEALTH_CHECK_PROXY_CLIENT_TIMEOUT.bindTo(flagSource) @@ -276,7 +279,7 @@ public class ModelContextImpl implements ModelContext { @Override public Duration jdiscHealthCheckProxyClientTimeout() { return Duration.ofMillis(jdiscHealthCheckProxyClientTimeout); } @Override public String jvmGCOptions() { return jvmGCOPtions; } - + @Override public String feedSequencerType() { return feedSequencer; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index bfdcc0e9c75..c35fcb3cf21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; @@ -72,6 +73,8 @@ public class SessionRepository { private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); private static final long nonExistingActiveSession = 0; + + private final SessionCache<LocalSession> localSessionCache = new SessionCache<>(); private final SessionCache<RemoteSession> remoteSessionCache = new SessionCache<>(); private final Map<Long, SessionStateWatcher> sessionStateWatchers = new HashMap<>(); @@ -89,6 +92,7 @@ public class SessionRepository { private final Path sessionsPath; private final TenantName tenantName; private final GlobalComponentRegistry componentRegistry; + private final Path locksPath; public SessionRepository(TenantName tenantName, GlobalComponentRegistry componentRegistry, @@ -109,6 +113,7 @@ public class SessionRepository { this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); this.reloadHandler = reloadHandler; this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + this.locksPath = TenantRepository.getLocksPath(tenantName); loadLocalSessions(); initializeRemoteSessions(); @@ -207,13 +212,15 @@ public class SessionRepository { public void deleteLocalSession(LocalSession session) { long sessionId = session.getSessionId(); - log.log(Level.FINE, "Deleting local session " + sessionId); - SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - localSessionCache.removeSession(sessionId); - NestedTransaction transaction = new NestedTransaction(); - deleteLocalSession(session, transaction); - transaction.commit(); + try (Lock lock = lock(sessionId)) { + log.log(Level.FINE, "Deleting local session " + sessionId); + SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + localSessionCache.removeSession(sessionId); + NestedTransaction transaction = new NestedTransaction(); + deleteLocalSession(session, transaction); + transaction.commit(); + } } /** Add transactions to delete this session to the given nested transaction */ @@ -335,7 +342,7 @@ public class SessionRepository { private void sessionRemoved(long sessionId) { SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); + if (watcher != null) watcher.close(); remoteSessionCache.removeSession(sessionId); metrics.incRemovedSessions(); } @@ -631,6 +638,15 @@ public class SessionRepository { public ReloadHandler getReloadHandler() { return reloadHandler; } + /** Returns the lock for session operations for the given session id. */ + public Lock lock(long sessionId) { + return curator.lock(lockPath(sessionId), Duration.ofMinutes(1)); // These locks shouldn't be held for very long. + } + + private Path lockPath(long sessionId) { + return locksPath.append(String.valueOf(sessionId)); + } + private static class FileTransaction extends AbstractTransaction { public static FileTransaction from(FileOperation operation) { diff --git a/container-jersey2/pom.xml b/container-jersey2/pom.xml index 78801126484..ee2c0f0cfab 100644 --- a/container-jersey2/pom.xml +++ b/container-jersey2/pom.xml @@ -40,6 +40,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-disc</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index bd9568fe891..24864c03530 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -16,9 +16,9 @@ public interface BillingController { PlanId getPlan(TenantName tenant); /** - * Returns true if plan was changed + * @return String containing error message if something went wrong. Empty otherwise */ - boolean setPlan(TenantName tenant, PlanId planId, boolean hasApplications); + PlanResult setPlan(TenantName tenant, PlanId planId, boolean hasApplications); Invoice.Id createInvoiceForPeriod(TenantName tenant, ZonedDateTime startTime, ZonedDateTime endTime, String agent); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index a4c25e301ba..4f367d6498e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -30,9 +30,9 @@ public class MockBillingController implements BillingController { } @Override - public boolean setPlan(TenantName tenant, PlanId planId, boolean hasApplications) { + public PlanResult setPlan(TenantName tenant, PlanId planId, boolean hasApplications) { plans.put(tenant, planId); - return true; + return PlanResult.success(); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanResult.java new file mode 100644 index 00000000000..a4803ccde64 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanResult.java @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import java.util.Optional; + +/** + * Result of {@link BillingController#setPlan} + * + * @author olaa + */ +public class PlanResult { + + private final Optional<String> errorMessage; + + private PlanResult(Optional<String> errorMessage) { + this.errorMessage = errorMessage; + } + + public static PlanResult success() { + return new PlanResult(Optional.empty()); + } + + public static PlanResult error(String errorMessage) { + return new PlanResult(Optional.of(errorMessage)); + } + + public boolean isSuccess() { + return errorMessage.isEmpty(); + } + + public Optional<String> getErrorMessage() { + return errorMessage; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java index 8f81c94257e..d3d60ecd64d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java @@ -21,6 +21,7 @@ public class WeightedAliasTarget extends AliasTarget { public WeightedAliasTarget(HostName name, String dnsZone, ZoneId zone, long weight) { super(name, dnsZone, zone.value()); this.weight = weight; + if (weight < 0) throw new IllegalArgumentException("Weight cannot be negative"); } /** The weight of this target */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index e1acf867744..7dd6126dabc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -70,7 +70,7 @@ public enum SystemApplication { if (controller.zoneRegistry().zones().reprovisionToUpgradeOs().ids().contains(zone)) { return nodeType == NodeType.host; // TODO(mpolden): Remove once all node types are supported } - return nodeType.isDockerHost(); + return nodeType.isHost(); } /** Returns whether this has an endpoint */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index ccbee15d2c5..0e6f856b115 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -139,12 +139,12 @@ public class BillingApiHandler extends LoggingRequestHandler { var planId = PlanId.from(slime.field("plan").asString()); var hasApplications = applicationController.asList(tenantName).size() > 0; - if (billingController.setPlan(tenantName, planId, hasApplications)) { + var result = billingController.setPlan(tenantName, planId, hasApplications); + + if (result.isSuccess()) return new StringResponse("Plan: " + planId.value()); - } else { - return ErrorResponse.forbidden("Invalid plan change with active deployments"); - } + return ErrorResponse.forbidden(result.getErrorMessage().orElse("Invalid plan change")); } private HttpResponse getBillingAllTenants(String until) { @@ -374,19 +374,6 @@ public class BillingApiHandler extends LoggingRequestHandler { return inspector.field(field).asString(); } - private DeploymentId getDeploymentIdOrNull(Inspector inspector) { - if (inspector.field("applicationId").valid() != inspector.field("zoneId").valid() ) { - throw new BadRequestException("Either both application id and zone id should be set, or neither."); - } - if (inspector.field("applicationId").valid()) { - return new DeploymentId( - ApplicationId.fromSerializedForm(inspector.field("applicationId").asString()), - com.yahoo.config.provision.zone.ZoneId.from(inspector.field("zoneId").asString()) - ); - } - return null; - } - private LocalDate untilParameter(String until) { if (until == null || until.isEmpty() || until.isBlank()) return LocalDate.now().plusDays(1); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 2fa931f2219..e5a99c2e69d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -177,51 +177,65 @@ public class RoutingPolicies { private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(routingPolicies); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - Map<RegionEndpoint, Set<AliasTarget>> targets = computeRegionEndpoints(routeEntry.getValue(), inactiveZones); + Collection<RegionEndpoint> regionEndpoints = computeRegionEndpoints(routeEntry.getValue(), inactiveZones); // Create a weighted ALIAS per region, pointing to all zones within the same region - targets.forEach(((regionEndpoint, weightedTargets) -> { - controller.nameServiceForwarder().createAlias(RecordName.from(regionEndpoint.dnsName), weightedTargets, + regionEndpoints.forEach(regionEndpoint -> { + controller.nameServiceForwarder().createAlias(RecordName.from(regionEndpoint.target().name().value()), + Collections.unmodifiableSet(regionEndpoint.zoneTargets()), Priority.normal); - })); + }); + // Create global latency-based ALIAS pointing to each per-region weighted ALIAS + Set<AliasTarget> latencyTargets = new LinkedHashSet<>(); + Set<AliasTarget> inactiveLatencyTargets = new LinkedHashSet<>(); + for (var regionEndpoint : regionEndpoints) { + if (regionEndpoint.active()) { + latencyTargets.add(regionEndpoint.target()); + } else { + inactiveLatencyTargets.add(regionEndpoint.target()); + } + } + // If all targets are configured out, all targets are set in. We do this because otherwise removing 100% of + // the ALIAS records would cause the global endpoint to stop resolving entirely (NXDOMAIN). + if (latencyTargets.isEmpty() && !inactiveLatencyTargets.isEmpty()) { + latencyTargets.addAll(inactiveLatencyTargets); + inactiveLatencyTargets.clear(); + } var endpoints = controller.routing().endpointsOf(routeEntry.getKey().application()) .named(routeEntry.getKey().endpointId()) .not().requiresRotation(); - Set<AliasTarget> latencyTargets = targets.keySet().stream() - .map(regionEndpoint -> new LatencyAliasTarget(HostName.from(regionEndpoint.dnsName), - regionEndpoint.dnsZone, - regionEndpoint.zone)) - .collect(Collectors.toSet()); endpoints.forEach(endpoint -> controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), latencyTargets, Priority.normal)); + inactiveLatencyTargets.forEach(t -> controller.nameServiceForwarder() + .removeRecords(Record.Type.ALIAS, + RecordData.fqdn(t.name().value()), + Priority.normal)); } } /** Compute region endpoints and their targets from given policies */ - private Map<RegionEndpoint, Set<AliasTarget>> computeRegionEndpoints(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) { - Map<RegionEndpoint, Set<AliasTarget>> targets = new LinkedHashMap<>(); + private Collection<RegionEndpoint> computeRegionEndpoints(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) { + Map<Endpoint, RegionEndpoint> endpoints = new LinkedHashMap<>(); RoutingMethod routingMethod = RoutingMethod.exclusive; for (var policy : policies) { if (policy.dnsZone().isEmpty()) continue; if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(routingMethod)) continue; Endpoint weighted = policy.weightedEndpointIn(controller.system(), routingMethod); - // Do not route to zone if global routing status is set out at: - // - zone level (ZoneRoutingPolicy) - // - deployment level (RoutingPolicy) - // - application package level (deployment.xml) - long weight = 1; var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); + long weight = 1; if (isConfiguredOut(policy, zonePolicy, inactiveZones)) { weight = 0; // A record with 0 weight will not received traffic. If all records within a group have 0 // weight, traffic is routed to all records with equal probability. } - var regionEndpoint = new RegionEndpoint(weighted, policy.dnsZone().get(), policy.id().zone()); var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone(), weight); - targets.computeIfAbsent(regionEndpoint, (k) -> new LinkedHashSet<>()) - .add(weightedTarget); + endpoints.computeIfAbsent(weighted, (k) -> new RegionEndpoint(new LatencyAliasTarget(HostName.from(weighted.dnsName()), + policy.dnsZone().get(), + policy.id().zone()))) + .zoneTargets() + .add(weightedTarget); } - return Collections.unmodifiableMap(targets); + return endpoints.values(); } /** Store routing policies for given load balancers */ @@ -335,18 +349,26 @@ public class RoutingPolicies { return false; } - /** Represents a region-wide endpoint */ + /** Represents records for a region-wide endpoint */ private static class RegionEndpoint { - private final String dnsName; - private final String dnsZone; - private final ZoneId zone; + private final LatencyAliasTarget target; + private final Set<WeightedAliasTarget> zoneTargets = new LinkedHashSet<>(); + + public RegionEndpoint(LatencyAliasTarget target) { + this.target = Objects.requireNonNull(target); + } + + public LatencyAliasTarget target() { + return target; + } + + public Set<WeightedAliasTarget> zoneTargets() { + return zoneTargets; + } - public RegionEndpoint(Endpoint endpoint, String dnsZone, ZoneId zone) { - this.dnsName = Objects.requireNonNull(endpoint).dnsName(); - this.dnsZone = Objects.requireNonNull(dnsZone); - this.zone = Objects.requireNonNull(zone); - if (endpoint.scope() != Endpoint.Scope.weighted) throw new IllegalArgumentException("Region endpoint must be weighted"); + public boolean active() { + return zoneTargets.stream().anyMatch(target -> target.weight() > 0); } @Override @@ -354,12 +376,12 @@ public class RoutingPolicies { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RegionEndpoint that = (RegionEndpoint) o; - return dnsName.equals(that.dnsName); + return target.name().equals(that.target.name()); } @Override public int hashCode() { - return Objects.hash(dnsName); + return Objects.hash(target.name()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index ca4bcf1a354..3ee4c6961a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -27,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -48,6 +48,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -55,6 +56,7 @@ import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -147,7 +149,7 @@ public class RoutingPoliciesTest { @Test public void global_routing_policies_with_duplicate_region() { var tester = new RoutingPoliciesTester(); - var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); int clustersPerZone = 2; int numberOfDeployments = 3; var applicationPackage = applicationPackageBuilder() @@ -157,15 +159,42 @@ public class RoutingPoliciesTest { .endpoint("r0", "c0") .endpoint("r1", "c1") .build(); - tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone3, zone4); + tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone3, zone4); // Creates alias records - context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone3, zone4); - tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 1, zone1, zone3, zone4); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone3, zone4); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 1, zone1, zone3, zone4); assertEquals("Routing policy count is equal to cluster count", numberOfDeployments * clustersPerZone, - tester.policiesOf(context1.instance().id()).size()); + tester.policiesOf(context.instance().id()).size()); + + // A zone in shared region is set out + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + + // Weight of inactive zone is set to zero + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L, + zone3, 1L, + zone4, 0L)); + + // Other zone in shared region is set out. Entire record group for the region is removed as all zones in the + // region are out (weight sum = 0) + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L)); + + // Everything is set back in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L, + zone3, 1L, + zone4, 1L)); } @Test @@ -395,9 +424,9 @@ public class RoutingPoliciesTest { GlobalRouting.Agent.tenant); context.flushDnsUpdates(); - // Inactive zone is given zero weight - tester.assertWeight(0, context.instanceId(), 0, zone1); - tester.assertWeight(1, context.instanceId(), 0, zone2); + // Inactive zone is removed from global DNS record + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); // Status details is stored in policy var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); @@ -414,15 +443,16 @@ public class RoutingPoliciesTest { // Next deployment does not affect status context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); context.flushDnsUpdates(); - tester.assertWeight(0, context.instanceId(), 0, zone1); - tester.assertWeight(1, context.instanceId(), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); // Deployment is set back in tester.controllerTester().clock().advance(Duration.ofHours(1)); changedAt = tester.controllerTester().clock().instant(); tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); context.flushDnsUpdates(); - tester.assertWeight(1, context.instanceId(), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); @@ -437,8 +467,8 @@ public class RoutingPoliciesTest { .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertWeight(1, context.instanceId(), 0, zone1); - tester.assertWeight(0, context.instanceId(), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); // ... back in var applicationPackage3 = applicationPackageBuilder() @@ -448,7 +478,8 @@ public class RoutingPoliciesTest { .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertWeight(1, context.instanceId(), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); } @Test @@ -468,14 +499,13 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); - tester.assertWeight(1, context.instanceId(), 0, zone1, zone2); } // Set zone out tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); context1.flushDnsUpdates(); - tester.assertWeight(1, context1.instanceId(), 0, zone1); - tester.assertWeight(0, context1.instanceId(), 0, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); for (var context : contexts) { var policies = tester.routingPolicies().get(context.instanceId()); assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, @@ -494,8 +524,8 @@ public class RoutingPoliciesTest { // Setting status per deployment does not affect status as entire zone is out tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); context1.flushDnsUpdates(); - tester.assertWeight(0, context1.instanceId(), 0, zone2); - tester.assertWeight(0, context2.instanceId(), 0, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); // Set single deployment out tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); @@ -504,9 +534,8 @@ public class RoutingPoliciesTest { // Set zone back in. Deployment set explicitly out, remains out, the rest are in tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); context1.flushDnsUpdates(); - tester.assertWeight(1, context1.instanceId(), 0, zone1); - tester.assertWeight(0, context1.instanceId(), 0, zone2); - tester.assertWeight(1, context2.instanceId(), 0, zone1, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } @Test @@ -549,7 +578,64 @@ public class RoutingPoliciesTest { // Deployment completes context.completeRollout(); - tester.assertTargets(context.instanceId(), endpointId, ClusterSpec.Id.from("default"), 0, prodZone); + tester.assertTargets(context.instanceId(), endpointId, ClusterSpec.Id.from("default"), 0, Map.of(prodZone, 1L)); + } + + @Test + public void changing_global_routing_status_never_removes_all_members() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Global DNS record is created, pointing to all configured zones + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Global routing status is overridden for one deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + + // Setting other deployment out implicitly sets all deployments in. Weight is set to zero, but that has no + // impact on routing decisions when the weight sum is zero + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); + + // One inactive deployment is put back in. Global DNS record now points to the only active deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Setting zone (containing active deployment) out puts all deployments in + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); + context.flushDnsUpdates(); + assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); + + // Setting zone back in removes the currently inactive deployment + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Inactive deployment is set in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { + assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); + } + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); } @Test @@ -584,7 +670,7 @@ public class RoutingPoliciesTest { lbHostname = HostName.from("shared-lb--" + zone.value()); } else { lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()); + "--" + zone.value()); } loadBalancers.add( new LoadBalancer("LB-" + i + "-Z-" + zone.value(), @@ -656,11 +742,11 @@ public class RoutingPoliciesTest { .collect(Collectors.toSet()); } - private List<String> aliasDataOf(String name) { + private Set<String> aliasDataOf(String name) { return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream() .map(Record::data) .map(RecordData::asString) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); } private List<String> cnameDataOf(String name) { @@ -670,27 +756,10 @@ public class RoutingPoliciesTest { .collect(Collectors.toList()); } - private void assertWeight(long expected, ApplicationId application, int loadBalancerId, ZoneId... zones) { - for (var zone : zones) { - Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone)) - .scope(Endpoint.Scope.weighted) - .named(EndpointId.of("c" + loadBalancerId)) - .asList() - .get(0); - List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, - RecordName.from(weighted.dnsName())); - assertEquals(1, records.size()); - assertEquals("Record " + weighted.dnsName() + " has expected weight", - expected, - WeightedAliasTarget.unpack(records.get(0).data()) - .weight()); - } - } - - private void assertTargets(ApplicationId application, EndpointId endpointId, ClusterSpec.Id clusterId, int loadBalancerId, ZoneId... zones) { + private void assertTargets(ApplicationId application, EndpointId endpointId, ClusterSpec.Id clusterId, int loadBalancerId, Map<ZoneId, Long> zoneWeights) { Set<String> latencyTargets = new HashSet<>(); Map<String, List<ZoneId>> zonesByRegionEndpoint = new HashMap<>(); - for (var zone : zones) { + for (var zone : zoneWeights.keySet()) { Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone)) .scope(Endpoint.Scope.weighted) .named(EndpointId.of(clusterId.value())) @@ -700,11 +769,11 @@ public class RoutingPoliciesTest { .add(zone); } zonesByRegionEndpoint.forEach((regionEndpoint, zonesInRegion) -> { - List<String> weightedTargets = zonesInRegion.stream() - .map(z -> "weighted/lb-" + loadBalancerId + "--" + - application.serializedForm() + "--" + z.value() + - "/dns-zone-1/" + z.value() + "/1") - .collect(Collectors.toList()); + Set<String> weightedTargets = zonesInRegion.stream() + .map(z -> "weighted/lb-" + loadBalancerId + "--" + + application.serializedForm() + "--" + z.value() + + "/dns-zone-1/" + z.value() + "/" + zoneWeights.get(z)) + .collect(Collectors.toSet()); assertEquals("Weighted endpoint " + regionEndpoint + " points to load balancer", weightedTargets, aliasDataOf(regionEndpoint)); @@ -714,7 +783,7 @@ public class RoutingPoliciesTest { }); String globalEndpoint = tester.controller().routing().endpointsOf(application) .named(endpointId) - .targets(List.of(zones)) + .targets(List.copyOf(zoneWeights.keySet())) .primary() .map(Endpoint::dnsName) .orElse("<none>"); @@ -724,7 +793,15 @@ public class RoutingPoliciesTest { } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId... zones) { - assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zones); + Map<ZoneId, Long> zoneWeights = new LinkedHashMap<>(); + for (var zone : zones) { + zoneWeights.put(zone, 1L); + } + assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zoneWeights); + } + + private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, Map<ZoneId, Long> zoneWeights) { + assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zoneWeights); } } diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 6bf676dcba1..4beaf6086a6 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -81,6 +81,11 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-disc</artifactId> <version>${project.version}</version> </dependency> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9737cba9291..4ab477dbaad 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -146,6 +146,12 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( + "feed-sequencer-type", "LATENCY", + "Selects type of sequenced executor used for feeding, valid values are LATENCY, ADAPTIVE, THROUGHPUT", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_DISTRIBUTOR_BTREE_DB = defineFeatureFlag( "use-distributor-btree-db", true, "Whether to use the new B-tree bucket database in the distributors.", diff --git a/hosted-tenant-base/is-base-pom-module.txt b/hosted-tenant-base/is-base-pom-module.txt new file mode 100644 index 00000000000..2b8523fd4bc --- /dev/null +++ b/hosted-tenant-base/is-base-pom-module.txt @@ -0,0 +1 @@ +Used to skip 'hosted-build-vespa-application' profile in 'hosted-tenant-base' when building 'cloud-tenant-base' itself
\ No newline at end of file diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 93ef008b8f7..03466255362 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -5,10 +5,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> - <!-- <version>7-SNAPSHOT</version>--> - <!-- <url>https://github.com/vespa-engine</url>--> - <!-- <packaging>pom</packaging>--> - <groupId>com.yahoo.vespa</groupId> <artifactId>hosted-tenant-base</artifactId> <version>7-SNAPSHOT</version> @@ -37,13 +33,28 @@ <properties> <vespaversion>${project.version}</vespaversion> - <test-framework.version>${project.version}</test-framework.version> <target_jdk_version>11</target_jdk_version> <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> - <junit.version>5.4.2</junit.version> - <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> + <junit.version>5.6.2</junit.version> <!-- NOTE: this must be in sync with junit version specified in 'tenant-cd-api' --> <test.categories>!integration</test.categories> + + <!-- To allow specialized base pom to include additional "test provided" dependencies --> + <bundlePluginExtraTestProvidedDependencies/> + + <!-- Must be specified by specialized base pom. Can also be overriden by application --> + <endpoint/> + + <!-- Properties that are specified by application pom --> + <tenant/> + <application/> + <instance/> + <environment/> + <region/> + <apiKeyFile/> + <apiCertificateFile/> + <dataPlaneKeyFile/> + <dataPlaneCertificateFile/> </properties> <dependencyManagement> @@ -60,12 +71,14 @@ <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> <version>${junit.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> <version>${junit.version}</version> + <scope>test</scope> </dependency> </dependencies> </dependencyManagement> @@ -82,139 +95,157 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-test</artifactId> <version>${vespaversion}</version> - <scope>runtime</scope> - <exclusions> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </exclusion> - </exclusions> + <scope>test</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>tenant-cd-api</artifactId> - <version>${test-framework.version}</version> + <version>${vespaversion}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> <scope>test</scope> </dependency> </dependencies> <profiles> <profile> - <!-- Build *-fat-test.jar file that includes all non-test classes and resources - that are part of the class path during test and and test.jar that includes - all test classes and resources, and put it inside a zip: - 1. application classes and resources - 2. test classes and resources - 3. classes and resources in all dependencies of both (1) and (2) - 4. copy the fat-test-jar and test-jar to application-test/artifacts directory - 5. zip application-test --> - <id>fat-test-application</id> + <id>hosted-build-vespa-application</id> + <activation> + <file> + <!-- don't run this profile when building this module --> + <missing>is-base-pom-module.txt</missing> + </file> + </activation> <build> + <finalName>${project.artifactId}</finalName> <plugins> + <plugin> - <!-- dependencies, see (3) above --> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <version>3.1.1</version> - <executions> - <execution> - <!-- JAR-like dependencies --> - <id>unpack-dependencies</id> - <phase>prepare-package</phase> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <configuration> - <includeTypes>jar,test-jar</includeTypes> - <outputDirectory>target/fat-test-classes</outputDirectory> - <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and - bcprov-jdk15on-1.58.jar are pulled in via - container-dev and both contains the same set of - bouncycastle signature files in META-INF: - BC1024KE.DSA, BC1024KE.SF, BC2048KE.DSA, and - BC2048KE.SF. By merging any of these two with any - other JAR file like we're doing here, the signatures - are wrong. Worse, what we're doing is WRONG but not - yet fatal. + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <groups>${test.categories}</groups> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <environment>${environment}</environment> + <region>${region}</region> + <endpoint>${endpoint}</endpoint> + <apiKeyFile>${apiKeyFile}</apiKeyFile> + <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> + <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> + <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> + </systemPropertyVariables> + </configuration> + </plugin> - The symptom of this happening is that the tester fails - to load the SystemTest class(!?), and subsequently - tries to run all test-like files in the fat test JAR. + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> - The solution is to exclude such files. This happens - automatically with maven-assembly-plugin. --> - <excludes>META-INF/*.SF,META-INF/*.DSA</excludes> - </configuration> - </execution> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>3.0.0-M2</version> + <executions> <execution> - <!-- non-JAR-like dependencies --> - <id>non-jar-dependencies</id> - <phase>prepare-package</phase> + <id>enforce-java</id> <goals> - <goal>copy-dependencies</goal> + <goal>enforce</goal> </goals> <configuration> - <excludeTypes>jar,test-jar</excludeTypes> - <outputDirectory>target/fat-test-classes</outputDirectory> + <rules> + <requireJavaVersion> + <version>[11, )</version> + </requireJavaVersion> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> </configuration> </execution> </executions> </plugin> + <plugin> - <artifactId>maven-resources-plugin</artifactId> - <version>3.1.0</version> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <version>${vespaversion}</version> + <extensions>true</extensions> + <configuration> + <testProvidedArtifacts> + ${bundlePluginExtraTestProvidedDependencies}, + com.yahoo.vespa:tenant-cd-api, + com.yahoo.vespa:container-test, + org.junit.jupiter:junit-jupiter-api, + org.junit.jupiter:junit-jupiter-engine, + !com.yahoo.vespa:application + </testProvidedArtifacts> + </configuration> <executions> <execution> - <id>copy-resources</id> + <id>generate-test-bundle-osgi-manifest</id> <phase>prepare-package</phase> <goals> - <goal>copy-resources</goal> + <goal>generate-test-bundle-osgi-manifest</goal> + </goals> + </execution> + <execution> + <id>assemble-test-bundle</id> + <phase>package</phase> + <goals> + <goal>assemble-test-bundle</goal> </goals> - <configuration> - <outputDirectory>target/fat-test-classes</outputDirectory> - <resources> - <!-- application classes and resources, see 1. above --> - <resource> - <directory>target/classes</directory> - </resource> - </resources> - </configuration> </execution> </executions> </plugin> + <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>3.1.0</version> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-maven-plugin</artifactId> + <version>${vespaversion}</version> <executions> <execution> - <id>fat-test-jar</id> - <phase>package</phase> + <id>generate-test-descriptor</id> + <phase>prepare-package</phase> <goals> - <goal>jar</goal> + <goal>generateTestDescriptor</goal> </goals> - <configuration> - <classesDirectory>target/fat-test-classes</classesDirectory> - <classifier>fat-test</classifier> - </configuration> </execution> + </executions> + </plugin> + + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> + <version>${vespaversion}</version> + <executions> <execution> - <id>test-jar</id> - <phase>package</phase> <goals> - <goal>test-jar</goal> + <goal>packageApplication</goal> </goals> </execution> </executions> </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> + <version>1.3</version> <executions> <execution> <id>attach-artifact</id> @@ -224,17 +255,8 @@ </goals> <configuration> <tasks> - <!-- copy fat test-jar to application-test artifacts directory, see 4. above --> - <copy file="target/${project.artifactId}-fat-test.jar" - todir="target/application-test/artifacts/" /> - - <!-- copy slim test-jar to application-test artifacts directory, see 4. above --> - <copy file="target/${project.artifactId}-tests.jar" - todir="target/application-test/artifacts/" /> - - <!-- zip application-test, see 5. above --> - <zip destfile="target/application-test.zip" - basedir="target/application-test/" /> + <copy file="target/${project.artifactId}-tests.jar" todir="target/application-test/artifacts/" /> + <zip destfile="target/application-test.zip" basedir="target/application-test/" /> </tasks> </configuration> </execution> @@ -270,70 +292,9 @@ </profiles> <build> - <finalName>${project.artifactId}</finalName> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <groups>${test.categories}</groups> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - <systemPropertyVariables> - <application>${application}</application> - <tenant>${tenant}</tenant> - <instance>${instance}</instance> - <environment>${environment}</environment> - <region>${region}</region> - <endpoint>${endpoint}</endpoint> - <apiKeyFile>${apiKeyFile}</apiKeyFile> - <apiCertificateFile>${apiCertificateFile}</apiCertificateFile> - <dataPlaneKeyFile>${dataPlaneKeyFile}</dataPlaneKeyFile> - <dataPlaneCertificateFile>${dataPlaneCertificateFile}</dataPlaneCertificateFile> - </systemPropertyVariables> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - </configuration> - </plugin> - </plugins> - </pluginManagement> - <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <version>3.0.0-M2</version> - <executions> - <execution> - <id>enforce-java</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireJavaVersion> - <version>[11, )</version> - </requireJavaVersion> - <requireMavenVersion> - <version>[3.5, )</version> - </requireMavenVersion> - </rules> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>${maven-compiler-plugin.version}</version> <configuration> @@ -347,42 +308,6 @@ </compilerArgs> </configuration> </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-maven-plugin</artifactId> - <version>${vespaversion}</version> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-application-maven-plugin</artifactId> - <version>${vespaversion}</version> - <executions> - <execution> - <goals> - <goal>packageApplication</goal> - </goals> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <version>${vespaversion}</version> - <extensions>true</extensions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - </plugin> </plugins> </build> </project> diff --git a/jdisc_messagebus_service/pom.xml b/jdisc_messagebus_service/pom.xml index eb41609a500..55f8392a5df 100644 --- a/jdisc_messagebus_service/pom.xml +++ b/jdisc_messagebus_service/pom.xml @@ -30,6 +30,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>component</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java index c910ac26833..f5f1eb5791d 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java @@ -56,7 +56,8 @@ public class ApplicationMetricsHandler extends HttpHandlerBase { @Override public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) { - if (apiPath.matches(METRICS_V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(METRICS_VALUES_PATH))); + if (apiPath.matches(METRICS_V1_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(METRICS_VALUES_PATH, + PROMETHEUS_VALUES_PATH))); if (apiPath.matches(METRICS_VALUES_PATH)) return Optional.of(applicationMetricsResponse(consumer)); if (apiPath.matches(PROMETHEUS_VALUES_PATH)) return Optional.of(applicationPrometheusResponse(consumer)); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index 46c4af4c1f4..cc6b6b36057 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -27,6 +27,7 @@ import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID; import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.METRICS_V1_PATH; import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.METRICS_VALUES_PATH; +import static ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler.PROMETHEUS_VALUES_PATH; import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; @@ -48,8 +49,9 @@ public class ApplicationMetricsHandlerTest { private static final String HOST = "localhost"; private static final String URI_BASE = "http://" + HOST; - private static final String APP_METRICS_V1_URI = URI_BASE + METRICS_V1_PATH; - private static final String APP_METRICS_VALUES_URI = URI_BASE + METRICS_VALUES_PATH; + private static final String METRICS_V1_URI = URI_BASE + METRICS_V1_PATH; + private static final String METRICS_VALUES_URI = URI_BASE + METRICS_VALUES_PATH; + private static final String PROMETHEUS_VALUES_URI = URI_BASE + PROMETHEUS_VALUES_PATH; private static final String TEST_FILE = "generic-sample.json"; private static final String RESPONSE = getFileContents(TEST_FILE); @@ -95,21 +97,23 @@ public class ApplicationMetricsHandlerTest { @Test public void v1_response_contains_values_uri() throws Exception { - String response = testDriver.sendRequest(APP_METRICS_V1_URI).readAll(); + String response = testDriver.sendRequest(METRICS_V1_URI).readAll(); JSONObject root = new JSONObject(response); assertTrue(root.has("resources")); JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); + assertEquals(2, resources.length()); JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(APP_METRICS_VALUES_URI, valuesUrl.getString("url")); + assertEquals(METRICS_VALUES_URI, valuesUrl.getString("url")); + JSONObject prometheusUrl = resources.getJSONObject(1); + assertEquals(PROMETHEUS_VALUES_URI, prometheusUrl.getString("url")); } @Ignore @Test public void visually_inspect_values_response() throws Exception { - String response = testDriver.sendRequest(APP_METRICS_VALUES_URI).readAll(); + String response = testDriver.sendRequest(METRICS_VALUES_URI).readAll(); ObjectMapper mapper = createObjectMapper(); var jsonModel = mapper.readValue(response, GenericApplicationModel.class); System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel)); @@ -166,13 +170,13 @@ public class ApplicationMetricsHandlerTest { @Test public void invalid_path_yields_error_response() throws Exception { - String response = testDriver.sendRequest(APP_METRICS_V1_URI + "/invalid").readAll(); + String response = testDriver.sendRequest(METRICS_V1_URI + "/invalid").readAll(); JSONObject root = new JSONObject(response); assertTrue(root.has("error")); } private GenericApplicationModel getResponseAsJsonModel(String consumer) { - String response = testDriver.sendRequest(APP_METRICS_VALUES_URI + "?consumer=" + consumer).readAll(); + String response = testDriver.sendRequest(METRICS_VALUES_URI + "?consumer=" + consumer).readAll(); try { return createObjectMapper().readValue(response, GenericApplicationModel.class); } catch (IOException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index b6237886dc7..be4f551ca29 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -175,7 +175,7 @@ public final class Node { * If both given wantToRetire and wantToDeprovision are equal to the current values, the method is no-op. */ public Node withWantToRetire(boolean wantToRetire, boolean wantToDeprovision, Agent agent, Instant at) { - if (!type.isDockerHost() && wantToDeprovision) + if (!type.isHost() && wantToDeprovision) throw new IllegalArgumentException("wantToDeprovision can only be set for hosts"); if (wantToRetire == status.wantToRetire() && wantToDeprovision == status.wantToDeprovision()) return this; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index c4c062311e6..0dd3869e234 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -518,7 +518,7 @@ public class NodeRepository extends AbstractComponent { new IllegalArgumentException("Could not deallocate " + hostname + ": Node not found")); List<Node> nodesToDirty = - (nodeToDirty.type().isDockerHost() ? + (nodeToDirty.type().isHost() ? Stream.concat(list().childrenOf(hostname).asList().stream(), Stream.of(nodeToDirty)) : Stream.of(nodeToDirty)) .filter(node -> node.state() != State.dirty) @@ -657,7 +657,7 @@ public class NodeRepository extends AbstractComponent { try (Mutex lock = lockUnallocated()) { requireRemovable(node, false, force); - if (node.type().isDockerHost()) { + if (node.type().isHost()) { List<Node> children = list().childrenOf(node).asList(); children.forEach(child -> requireRemovable(child, true, force)); db.removeNodes(children); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 7e02eedc572..d65b4ce4248 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -100,7 +100,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { List<Node> nodesToRecycle = new ArrayList<>(); for (Node candidate : nodes) { if (NodeFailer.hasHardwareIssue(candidate, nodeRepository)) { - List<String> unparkedChildren = !candidate.type().isDockerHost() ? Collections.emptyList() : + List<String> unparkedChildren = !candidate.type().isHost() ? Collections.emptyList() : nodeRepository.list().childrenOf(candidate).asList().stream() .filter(node -> node.state() != Node.State.parked) .map(Node::hostname) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 1e978682fa9..9c1892a1920 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -102,7 +102,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { for (Map.Entry<Node, String> entry : getReadyNodesByFailureReason().entrySet()) { Node node = entry.getKey(); if (throttle(node)) { - if (node.type().isDockerHost()) throttledHostFailures++; + if (node.type().isHost()) throttledHostFailures++; else throttledNodeFailures++; continue; } @@ -117,12 +117,14 @@ public class NodeFailer extends NodeRepositoryMaintainer { // Fail active nodes for (Map.Entry<Node, String> entry : getActiveNodesByFailureReason(activeNodes).entrySet()) { Node node = entry.getKey(); - if (!failAllowedFor(node.type())) { - continue; - } + if (!failAllowedFor(node.type())) continue; + if (throttle(node)) { - if (node.type().isDockerHost()) throttledHostFailures++; - else throttledNodeFailures++; + if (node.type().isHost()) + throttledHostFailures++; + else + throttledNodeFailures++; + continue; } String reason = entry.getValue(); @@ -212,9 +214,10 @@ public class NodeFailer extends NodeRepositoryMaintainer { for (Node node : activeNodes) { if (node.history().hasEventBefore(History.Event.Type.down, graceTimeEnd) && ! applicationSuspended(node)) { nodesByFailureReason.put(node, "Node has been down longer than " + downTimeLimit); - } else if (hostSuspended(node, activeNodes)) { + } + else if (hostSuspended(node, activeNodes)) { Node hostNode = node.parentHostname().flatMap(parent -> nodeRepository().getNode(parent)).orElse(node); - if (hostNode.type().isDockerHost()) { + if (hostNode.type().isHost()) { List<String> failureReports = reasonsToFailParentHost(hostNode); if (failureReports.size() > 0) { if (hostNode.equals(node)) { @@ -244,7 +247,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { } private boolean expectConfigRequests(Node node) { - return !node.type().isDockerHost(); + return !node.type().isHost(); } private boolean hasNodeRequestedConfigAfter(Node node, Instant instant) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java index a36ef8fda4d..11afbd785e8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java @@ -24,7 +24,7 @@ public class OsUpgradeActivator extends NodeRepositoryMaintainer { @Override protected void maintain() { for (var nodeType : NodeType.values()) { - if (!nodeType.isDockerHost()) continue; + if (!nodeType.isHost()) continue; var active = canUpgradeOsOf(nodeType); nodeRepository().osVersions().resumeUpgradeOf(nodeType, active); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index 73869fb098d..a8566e24743 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -85,7 +85,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { * - Orchestrator allows it */ private boolean canRemove(Node node) { - if (node.type().isDockerHost()) { + if (node.type().isHost()) { if (nodeRepository() .list().childrenOf(node).asList().stream() .allMatch(child -> child.state() == Node.State.parked || diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 3210de18b7d..836583022ca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -133,7 +133,7 @@ public class IP { var addresses = new HashSet<>(node.ipConfig().primary()); var otherAddresses = new HashSet<>(other.ipConfig().primary()); - if (node.type().isDockerHost()) { // Addresses of a host can never overlap with any other nodes + if (node.type().isHost()) { // Addresses of a host can never overlap with any other nodes addresses.addAll(node.ipConfig().pool().asSet()); otherAddresses.addAll(other.ipConfig().pool().asSet()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index 54586105720..8e38aabf7ce 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -126,7 +126,7 @@ public class OsVersions { } private static void require(NodeType nodeType) { - if (!nodeType.isDockerHost()) { + if (!nodeType.isHost()) { throw new IllegalArgumentException("Node type '" + nodeType + "' does not support OS upgrades"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java index b2b83b6d064..aebf14ab13f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringUpgrader.java @@ -60,7 +60,7 @@ public class RetiringUpgrader implements Upgrader { /** Retire and deprovision given host and its children */ private void retire(Node host, Version target, Instant now) { - if (!host.type().isDockerHost()) throw new IllegalArgumentException("Cannot retire non-host " + host); + if (!host.type().isHost()) throw new IllegalArgumentException("Cannot retire non-host " + host); try (var lock = nodeRepository.lock(host)) { Optional<Node> currentNode = nodeRepository.getNode(host.hostname()); if (currentNode.isEmpty()) return; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 367271564ea..a8086400e26 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -257,7 +257,7 @@ public class CuratorDatabaseClient implements JobControl.Db { /** Returns whether to reboot node as part of transition to given state. This is done to get rid of any lingering * unwanted state (e.g. processes) on non-host nodes. */ private boolean rebootOnTransitionTo(Node.State state, Node node) { - if (node.type().isDockerHost()) return false; // Reboot of host nodes is handled by NodeRebooter + if (node.type().isHost()) return false; // Reboot of host nodes is handled by NodeRebooter if (zone.environment().isTest()) return false; // We want to reuse nodes quickly in test environments return node.state() != Node.State.dirty && state == Node.State.dirty; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java index 9edcfd6c697..3f55307d1f3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java @@ -60,13 +60,13 @@ public class DockerImages { /** Returns the current docker image for given node type, or the type for corresponding child nodes * if it is a Docker host, or default */ public DockerImage dockerImageFor(NodeType type) { - NodeType typeToUseForLookup = type.isDockerHost() ? type.childNodeType() : type; + NodeType typeToUseForLookup = type.isHost() ? type.childNodeType() : type; return getDockerImages().getOrDefault(typeToUseForLookup, defaultImage); } /** Set the docker image for nodes of given type */ public void setDockerImage(NodeType nodeType, Optional<DockerImage> dockerImage) { - if (nodeType.isDockerHost()) { + if (nodeType.isHost()) { throw new IllegalArgumentException("Setting docker image for " + nodeType + " nodes is unsupported"); } try (Lock lock = db.lockDockerImages()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 897af634d49..57f9b217d73 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -96,7 +96,7 @@ public class NodePatcher { private List<Node> applyFieldRecursive(String name, Inspector value, Inspector root) { switch (name) { case WANT_TO_RETIRE: - List<Node> childNodes = node.type().isDockerHost() ? nodes.get().childrenOf(node).asList() : List.of(); + List<Node> childNodes = node.type().isHost() ? nodes.get().childrenOf(node).asList() : List.of(); return childNodes.stream() .map(child -> applyField(child, name, value, root)) .collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index ce9a461e17f..58e838018c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -172,7 +172,7 @@ class NodesResponse extends HttpResponse { node.status().osVersion().current().ifPresent(version -> object.setString("currentOsVersion", version.toFullString())); node.status().osVersion().wanted().ifPresent(version -> object.setString("wantedOsVersion", version.toFullString())); node.status().firmwareVerifiedAt().ifPresent(instant -> object.setLong("currentFirmwareCheck", instant.toEpochMilli())); - if (node.type().isDockerHost()) + if (node.type().isHost()) nodeRepository.firmwareChecks().requiredAfter().ifPresent(after -> object.setLong("wantedFirmwareCheck", after.toEpochMilli())); node.status().vespaVersion().ifPresent(version -> object.setString("vespaVersion", version.toFullString())); currentDockerImage(node).ifPresent(dockerImage -> object.setString("currentDockerImage", dockerImage.asString())); diff --git a/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt b/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt index d1d566b1cab..150db076803 100644 --- a/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt +++ b/searchlib/src/tests/queryeval/wrappers/CMakeLists.txt @@ -1,11 +1,12 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchlib_wrappers_test_app TEST SOURCES wrappers_test.cpp DEPENDS searchlib searchlib_test - gtest + GTest::GTest ) vespa_add_test(NAME searchlib_wrappers_test_app COMMAND searchlib_wrappers_test_app) diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp index 3e87749c794..765d7b6c9f5 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp @@ -1,6 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "adaptive_sequenced_executor.h" +#include <vespa/vespalib/util/lambdatask.h> namespace vespalib { @@ -218,6 +219,35 @@ AdaptiveSequencedExecutor::worker_main() _thread_tools->allow_worker_exit.await(); } +void +AdaptiveSequencedExecutor::run_task_in_strand(Task::UP task, Strand &strand, std::unique_lock<std::mutex> &lock) +{ + assert(_self.state != Self::State::CLOSED); + strand.queue.push(std::move(task)); + _stats.queueSize.add(++_self.pending_tasks); + ++_stats.acceptedTasks; + if (strand.state == Strand::State::WAITING) { + ++_self.waiting_tasks; + } else if (strand.state == Strand::State::IDLE) { + if (_worker_stack.size() < _cfg.num_threads) { + strand.state = Strand::State::WAITING; + _wait_queue.push(&strand); + _self.waiting_tasks += strand.queue.size(); + } else { + strand.state = Strand::State::ACTIVE; + assert(_wait_queue.empty()); + Worker *worker = _worker_stack.back(); + _worker_stack.popBack(); + assert(worker->state == Worker::State::BLOCKED); + assert(worker->strand == nullptr); + worker->state = Worker::State::RUNNING; + worker->strand = &strand; + lock.unlock(); // UNLOCK + worker->cond.notify_one(); + } + } +} + AdaptiveSequencedExecutor::AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, size_t max_waiting, size_t max_pending) : ISequencedTaskExecutor(num_strands), @@ -268,38 +298,23 @@ AdaptiveSequencedExecutor::executeTask(ExecutorId id, Task::UP task) Strand &strand = _strands[id.getId()]; auto guard = std::unique_lock(_mutex); maybe_block_self(guard); - assert(_self.state != Self::State::CLOSED); - strand.queue.push(std::move(task)); - _stats.queueSize.add(++_self.pending_tasks); - ++_stats.acceptedTasks; - if (strand.state == Strand::State::WAITING) { - ++_self.waiting_tasks; - } else if (strand.state == Strand::State::IDLE) { - if (_worker_stack.size() < _cfg.num_threads) { - strand.state = Strand::State::WAITING; - _wait_queue.push(&strand); - _self.waiting_tasks += strand.queue.size(); - } else { - strand.state = Strand::State::ACTIVE; - assert(_wait_queue.empty()); - Worker *worker = _worker_stack.back(); - _worker_stack.popBack(); - assert(worker->state == Worker::State::BLOCKED); - assert(worker->strand == nullptr); - worker->state = Worker::State::RUNNING; - worker->strand = &strand; - guard.unlock(); // UNLOCK - worker->cond.notify_one(); - } - } + run_task_in_strand(std::move(task), strand, guard); } void AdaptiveSequencedExecutor::sync() { vespalib::CountDownLatch latch(_strands.size()); - for (size_t i = 0; i < _strands.size(); ++i) { - execute(ExecutorId(i), [&](){ latch.countDown(); }); + { + auto guard = std::unique_lock(_mutex); + for (Strand &strand: _strands) { + if (strand.state == Strand::State::IDLE) { + latch.countDown(); + } else { + Task::UP task = vespalib::makeLambdaTask([&](){ latch.countDown(); }); + run_task_in_strand(std::move(task), strand, guard); + } + } } latch.await(); } diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index a4d3ac97758..8f8b359797e 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h @@ -113,6 +113,7 @@ private: bool exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock); Task::UP next_task(Worker &worker); void worker_main(); + void run_task_in_strand(Task::UP task, Strand &strand, std::unique_lock<std::mutex> &lock); public: AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, size_t max_waiting, size_t max_pending); diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 4dbeb5a9a22..fcb019fb6bc 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -33,7 +33,7 @@ namespace storage { BucketManager::BucketManager(const config::ConfigUri & configUri, ServiceLayerComponentRegister& compReg) - : StorageLinkQueued("Bucket manager", compReg), + : StorageLink("Bucket manager"), framework::StatusReporter("bucketdb", "Bucket database"), _configUri(configUri), _workerLock(), @@ -85,7 +85,6 @@ void BucketManager::onClose() _thread->interruptAndJoin(_workerLock, _workerCond); _thread.reset(); } - StorageLinkQueued::onClose(); } void @@ -417,11 +416,6 @@ void BucketManager::startWorkerThread() _thread = _component.startThread(*this, maxProcessingTime, waitTime); } -void BucketManager::onFlush(bool downwards) -{ - StorageLinkQueued::onFlush(downwards); -} - // --------- Commands --------- // bool BucketManager::onRequestBucketInfo( @@ -471,7 +465,7 @@ bool BucketManager::onRequestBucketInfo( entry._bucketId.toString().c_str(), entry._info.toString().c_str()); } - dispatchUp(reply); + sendUp(reply); // Remaining replies dispatched by queueGuard upon function exit. return true; } @@ -527,7 +521,7 @@ BucketManager::leaveQueueProtectedSection(ScopedQueueDispatchGuard& queueGuard) --_requestsCurrentlyProcessing; if (_requestsCurrentlyProcessing == 0) { for (auto& qr : _queuedReplies) { - dispatchUp(qr); + sendUp(qr); } _queuedReplies.clear(); _conflictingBuckets.clear(); @@ -618,7 +612,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac LOG(debug, "Rejecting request from distributor %u: %s", (*it)->getDistributor(), error.str().c_str()); - dispatchUp(reply); + sendUp(reply); } if (requests.empty()) { @@ -669,7 +663,7 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac auto reply(std::make_shared<api::RequestBucketInfoReply>( *nodeAndCmd.second)); reply->getBucketInfo().swap(result[nodeAndCmd.first]); - dispatchUp(reply); + sendUp(reply); } reqs.clear(); @@ -689,7 +683,7 @@ bool BucketManager::onUp(const std::shared_ptr<api::StorageMessage>& msg) { if (!StorageLink::onUp(msg)) { - dispatchUp(msg); + sendUp(msg); } return true; } diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index cb534ad4002..9ba5f770f9d 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -27,7 +27,7 @@ namespace storage { -class BucketManager : public StorageLinkQueued, +class BucketManager : public StorageLink, public framework::StatusReporter, private framework::Runnable, private framework::MetricUpdateHook @@ -40,8 +40,6 @@ public: private: config::ConfigUri _configUri; - - uint32_t _chunkLevel; BucketInfoRequestMap _bucketInfoRequests; /** @@ -122,7 +120,6 @@ private: void onOpen() override; void onDoneInit() override { _doneInitialized = true; } void onClose() override; - void onFlush(bool downwards) override; void updateMetrics(bool updateDocCount); void updateMetrics(const MetricLockGuard &) override { updateMetrics(true); } diff --git a/storage/src/vespa/storage/common/messagesender.cpp b/storage/src/vespa/storage/common/messagesender.cpp index 34a892ffe04..3dfbbb67d31 100644 --- a/storage/src/vespa/storage/common/messagesender.cpp +++ b/storage/src/vespa/storage/common/messagesender.cpp @@ -16,4 +16,9 @@ MessageSender::send(const std::shared_ptr<api::StorageMessage>& msg) } } +void +MessageSender::sendReplyDirectly(const std::shared_ptr<api::StorageReply>& reply) { + sendReply(reply); +} + } diff --git a/storage/src/vespa/storage/common/messagesender.h b/storage/src/vespa/storage/common/messagesender.h index 2f24b750d66..48020bf053e 100644 --- a/storage/src/vespa/storage/common/messagesender.h +++ b/storage/src/vespa/storage/common/messagesender.h @@ -31,6 +31,8 @@ struct MessageSender { virtual void sendCommand(const std::shared_ptr<api::StorageCommand>&) = 0; virtual void sendReply(const std::shared_ptr<api::StorageReply>&) = 0; + // By calling this you certify that it can continue in same thread or be dispatched. + virtual void sendReplyDirectly(const std::shared_ptr<api::StorageReply>&); void send(const std::shared_ptr<api::StorageMessage>&); }; diff --git a/storage/src/vespa/storage/common/storagelinkqueued.h b/storage/src/vespa/storage/common/storagelinkqueued.h index 9fa0099f9a1..25b6c4fd1b6 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.h +++ b/storage/src/vespa/storage/common/storagelinkqueued.h @@ -128,8 +128,6 @@ private: }; framework::ComponentRegister& _compReg; - std::unique_ptr<framework::Thread> _replyThread; - std::unique_ptr<framework::Thread> _commandThread; ReplyDispatcher _replyDispatcher; CommandDispatcher _commandDispatcher; uint16_t _closeState; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 44cd8f0ab0c..d5aecaa6da0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -104,6 +104,12 @@ FileStorHandler::sendReply(const api::StorageReply::SP& msg) } void +FileStorHandler::sendReplyDirectly(const api::StorageReply::SP& msg) +{ + _impl->sendReplyDirectly(msg); +} + +void FileStorHandler::getStatus(std::ostream& out, const framework::HttpUrlPath& path) const { _impl->getStatus(out, path); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index af0a52b2fa0..973e51b04e3 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -239,6 +239,7 @@ public: void sendCommand(const api::StorageCommand::SP&) override; /** Send the given reply back out of the persistence layer. */ void sendReply(const api::StorageReply::SP&) override; + void sendReplyDirectly(const std::shared_ptr<api::StorageReply>&) override; /** Writes status page. */ void getStatus(std::ostream& out, const framework::HttpUrlPath& path) const; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index fbdbac27b7c..f536def28a0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -837,6 +837,12 @@ FileStorHandlerImpl::sendReply(const std::shared_ptr<api::StorageReply>& msg) _messageSender.sendReply(msg); } +void +FileStorHandlerImpl::sendReplyDirectly(const std::shared_ptr<api::StorageReply>& msg) +{ + _messageSender.sendReplyDirectly(msg); +} + FileStorHandlerImpl::MessageEntry::MessageEntry(const std::shared_ptr<api::StorageMessage>& cmd, const document::Bucket &bucket) : _command(cmd), diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 00714c291b7..a609473afd5 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -254,6 +254,7 @@ public: } void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; void sendReply(const std::shared_ptr<api::StorageReply>&) override; + void sendReplyDirectly(const api::StorageReply::SP& msg) override; void getStatus(std::ostream& out, const framework::HttpUrlPath& path) const; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 021c94464df..b2a06dc5633 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -769,6 +769,19 @@ FileStorManager::sendReply(const std::shared_ptr<api::StorageReply>& reply) } void +FileStorManager::sendReplyDirectly(const std::shared_ptr<api::StorageReply>& reply) +{ + LOG(spam, "Sending reply %s", reply->toString().c_str()); + + if (reply->getType() == api::MessageType::INTERNAL_REPLY) { + std::shared_ptr<api::InternalReply> rep(std::dynamic_pointer_cast<api::InternalReply>(reply)); + assert(rep.get()); + if (onInternalReply(rep)) return; + } + sendUp(reply); +} + +void FileStorManager::sendUp(const std::shared_ptr<api::StorageMessage>& msg) { StorageLinkQueued::sendUp(msg); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 452b83bb794..ba3d378b9e6 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -149,6 +149,7 @@ private: void handleAbortBucketOperations(const std::shared_ptr<AbortBucketOperationsCommand>&); void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; void sendReply(const std::shared_ptr<api::StorageReply>&) override; + void sendReplyDirectly(const std::shared_ptr<api::StorageReply>&) override; void sendUp(const std::shared_ptr<api::StorageMessage>&) override; void onClose() override; void onFlush(bool downwards) override; diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 72d6a1bb8d3..6605e3f6363 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -99,7 +99,7 @@ MessageTracker::sendReply() { } LOG(spam, "Sending reply up: %s %" PRIu64, getReply().toString().c_str(), getReply().getMsgId()); - _replySender.sendReply(std::move(_reply)); + _replySender.sendReplyDirectly(std::move(_reply)); } else { if ( ! _context.getTrace().getRoot().isEmpty()) { _msg->getTrace().getRoot().addChild(_context.getTrace().getRoot()); diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml index 23e5f3ec3f4..47e0f6e3fe6 100644 --- a/tenant-cd-api/pom.xml +++ b/tenant-cd-api/pom.xml @@ -19,6 +19,16 @@ <relativePath>../parent</relativePath> </parent> + <properties> + + <!-- + This version must match the string in all ExportPackage annotations. + It must also be in sync junit version specified in 'hosted-tenant-base'. + --> + <hosted-tenant-base-junit-version>5.6.2</hosted-tenant-base-junit-version> + + </properties> + <dependencies> <!-- provided --> <dependency> @@ -42,10 +52,10 @@ </dependency> <!-- compile --> - <dependency> <!-- TODO(bjorncs): share junit version number with test-runner implementation --> + <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> - <version>5.6.2</version> <!-- NOTE: This version must match the string in all ExportPackage annotations --> + <version>${hosted-tenant-base-junit-version}</version> <scope>compile</scope> </dependency> </dependencies> diff --git a/vespalog/src/vespa-logfmt/vespa-logfmt.pl b/vespalog/src/vespa-logfmt/vespa-logfmt.pl index f15a44d4419..78e06aa5845 100755 --- a/vespalog/src/vespa-logfmt/vespa-logfmt.pl +++ b/vespalog/src/vespa-logfmt/vespa-logfmt.pl @@ -50,6 +50,7 @@ my $msgtxre; my $onlypid; my $onlysvc; my $onlyhst; +my $onlyint; my $shortsvc; my $shortcmp; @@ -67,6 +68,7 @@ GetOptions ('level|l=s' => \@optlevels, 'service|S=s' => \$onlysvc, 'show|s=s' => \@optshow, 'pid|p=s' => \$onlypid, + 'internal|i' => \$onlyint, 'component|c=s' => \$compore, 'message|m=s' => \$msgtxre, 'help|h' => \$opthelp, @@ -107,6 +109,7 @@ if ( $opthelp || $bad ) { " -N\t\t--nldequote\t\tdequote newlines in message text field\n", " -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n", " --ts\t\t--truncateservice\tchop service to 9 chars\n", + " -i\t\t--internal\t\tfilter out plugin-generated messages\n", "\n", "FIELDLIST is comma separated, available fields:\n", "\t time fmttime msecs usecs host level pid service component message\n", @@ -172,6 +175,31 @@ if ( $optshow ) { # foreach ( keys %showflags ) ; } +my $only_internal_regexp = qr/^ + [^.]*[.] + ( ai[.]vespa[.] | + com[.]yahoo[.] + ( application | binaryprefix | clientmetrics | + collections | component | compress | + concurrent | config | configtest | + container | data | docproc | docprocs | + document | documentapi | documentmodel | + dummyreceiver | errorhandling | exception | + feedapi | feedhandler | filedistribution | + fs4 | fsa | geo | io | javacc | jdisc | + jrt | lang | language | log | logserver | + messagebus | metrics | net | osgi | path | + plugin | prelude | processing | protect | + reflection | restapi | search | + searchdefinition | searchlib | security | + slime | socket | statistics | stream | + system | tensor | test | text | + time | transaction | vdslib | vespa | + vespaclient | vespafeeder | vespaget | + vespastat | vespasummarybenchmark | + vespavisit | vespaxmlparser | yolean ) + )[.]/x ; + while (<>) { chomp; if ( /^ @@ -205,6 +233,12 @@ while (<>) { } next unless ( $levelflags{$levl} ); + # for now, only filter plugins in "Container" + if ($onlyint && $comp =~ m/^Container[.]/) { + if ($comp !~ m/$only_internal_regexp/) { + next; + } + } if ($compore && $comp !~ m/$compore/o) { next; } if ($msgtxre && $msgt !~ m/$msgtxre/o) { next; } if ($onlypid && $pidn ne $onlypid) { next; } |